From 3bcb78b3aff565996ee0e2aa96bce7f1bdd6d66a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 24 Nov 2015 08:33:59 +0100 Subject: [PATCH] Make the network auto sufficient (eject bad pods with scores) --- README.md | 2 +- config/test-4.yaml | 19 +++ config/test-5.yaml | 20 +++ config/test-6.yaml | 21 ++++ package.json | 2 + scripts/clean_test.sh | 7 +- server.js | 10 +- src/database.js | 3 +- src/pods.js | 64 ++++++++-- src/utils.js | 30 +++-- src/videos.js | 21 +--- test/api/friendsAdvanced.js | 154 +++++++++++++++++++++++ test/api/{friends.js => friendsBasic.js} | 2 +- test/utils.js | 4 +- 14 files changed, 310 insertions(+), 49 deletions(-) create mode 100644 config/test-4.yaml create mode 100644 config/test-5.yaml create mode 100644 config/test-6.yaml create mode 100644 test/api/friendsAdvanced.js rename test/api/{friends.js => friendsBasic.js} (98%) diff --git a/README.md b/README.md index 1a738539a..9f028f40e 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Thanks to [webtorrent](https://github.com/feross/webtorrent), we can make P2P (t - [ ] Inscription - [ ] Connection - [ ] Account rights (upload...) -- [ ] Make the network auto sufficient (eject bad pods etc) +- [X] Make the network auto sufficient (eject bad pods etc) - [ ] Manage API breaks - [ ] Add "DDOS" security (check if a pod don't send too many requests for example) diff --git a/config/test-4.yaml b/config/test-4.yaml new file mode 100644 index 000000000..6db8a5d03 --- /dev/null +++ b/config/test-4.yaml @@ -0,0 +1,19 @@ +listen: + port: 9004 + +webserver: + host: 'localhost' + port: 9004 + +database: + suffix: '-test4' + +# From the project root directory +storage: + certs: 'test4/certs/' + uploads: 'test4/uploads/' + logs: 'test4/logs/' + +network: + friends: + - 'http://localhost:9002' diff --git a/config/test-5.yaml b/config/test-5.yaml new file mode 100644 index 000000000..7b3f18d35 --- /dev/null +++ b/config/test-5.yaml @@ -0,0 +1,20 @@ +listen: + port: 9005 + +webserver: + host: 'localhost' + port: 9005 + +database: + suffix: '-test5' + +# From the project root directory +storage: + certs: 'test5/certs/' + uploads: 'test5/uploads/' + logs: 'test5/logs/' + +network: + friends: + - 'http://localhost:9001' + - 'http://localhost:9004' diff --git a/config/test-6.yaml b/config/test-6.yaml new file mode 100644 index 000000000..0c7675cc2 --- /dev/null +++ b/config/test-6.yaml @@ -0,0 +1,21 @@ +listen: + port: 9006 + +webserver: + host: 'localhost' + port: 9006 + +database: + suffix: '-test6' + +# From the project root directory +storage: + certs: 'test6/certs/' + uploads: 'test6/uploads/' + logs: 'test6/logs/' + +network: + friends: + - 'http://localhost:9001' + - 'http://localhost:9002' + - 'http://localhost:9003' diff --git a/package.json b/package.json index 0c7af5690..86c832eb1 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,9 @@ "confirm", "it", "after", + "afterEach", "before", + "beforeEach", "describe" ] } diff --git a/scripts/clean_test.sh b/scripts/clean_test.sh index 8868cbddf..e46b5ecd5 100755 --- a/scripts/clean_test.sh +++ b/scripts/clean_test.sh @@ -1,5 +1,6 @@ #!/bin/bash -printf "use peertube-test1;\ndb.dropDatabase();\nuse peertube-test2;\ndb.dropDatabase();\nuse peertube-test3;\ndb.dropDatabase();" | mongo - -rm -rf ./test1 ./test2 ./test3 +for i in $(seq 1 6); do + printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo + rm -rf "./test$i" +done diff --git a/server.js b/server.js index 715556414..3b899689c 100644 --- a/server.js +++ b/server.js @@ -1,9 +1,6 @@ ;(function () { 'use strict' - // ----------- Constants ----------- - global.API_VERSION = 'v1' - // ----------- Node modules ----------- var bodyParser = require('body-parser') var express = require('express') @@ -30,11 +27,16 @@ checker.createDirectoriesIfNotExist() + // ----------- Constants ----------- + var utils = require('./src/utils') + + global.API_VERSION = 'v1' + global.FRIEND_BASE_SCORE = utils.isTestInstance() ? 20 : 100 + // ----------- PeerTube modules ----------- var config = require('config') var logger = require('./src/logger') var routes = require('./routes') - var utils = require('./src/utils') var videos = require('./src/videos') var webtorrent = require('./src/webTorrentNode') diff --git a/src/database.js b/src/database.js index 020bfd961..740e89fa4 100644 --- a/src/database.js +++ b/src/database.js @@ -24,7 +24,8 @@ // ----------- Pods ----------- var podsSchema = mongoose.Schema({ url: String, - publicKey: String + publicKey: String, + score: { type: Number, max: global.FRIEND_BASE_SCORE } }) var PodsDB = mongoose.model('pods', podsSchema) diff --git a/src/pods.js b/src/pods.js index b4325ebcf..e26b3f0ae 100644 --- a/src/pods.js +++ b/src/pods.js @@ -16,6 +16,13 @@ var host = config.get('webserver.host') var port = config.get('webserver.port') + // ----------- Constants ----------- + + var PODS_SCORE = { + MALUS: -10, + BONUS: 10 + } + // ----------- Private functions ----------- function getForeignPodsList (url, callback) { @@ -27,6 +34,25 @@ }) } + function updatePodsScore (good_pods, bad_pods) { + logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length) + + PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec() + PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) { + if (err) throw err + removeBadPods() + }) + } + + function removeBadPods () { + PodsDB.remove({ score: 0 }, function (err, result) { + if (err) throw err + + var number_removed = result.result.n + if (number_removed !== 0) logger.info('Removed %d pod.', number_removed) + }) + } + // ----------- Public functions ----------- pods.list = function (callback) { @@ -46,7 +72,8 @@ var params = { url: data.url, - publicKey: data.publicKey + publicKey: data.publicKey, + score: global.FRIEND_BASE_SCORE } PodsDB.create(params, function (err, pod) { @@ -68,7 +95,9 @@ // { path, data } pods.makeSecureRequest = function (data, callback) { - PodsDB.find({}, { url: 1, publicKey: 1 }).exec(function (err, urls) { + if (callback === undefined) callback = function () {} + + PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) { if (err) { logger.error('Cannot get the list of the pods.', { error: err }) return callback(err) @@ -84,15 +113,23 @@ data: data.data } + var bad_pods = [] + var good_pods = [] + utils.makeMultipleRetryRequest( params, - urls, + pods, - function callbackEachPodFinished (err, response, body, url) { + function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) { if (err || response.statusCode !== 200) { - logger.error('Error sending secure request to %s/%s pod.', url, data.path, { error: err }) + bad_pods.push(pod._id) + logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err }) + } else { + good_pods.push(pod._id) } + + return callback_each_pod_finished() }, function callbackAllPodsFinished (err) { @@ -102,6 +139,8 @@ } logger.debug('Finished') + + updatePodsScore(good_pods, bad_pods) callback(null) } ) @@ -133,8 +172,8 @@ // ----------------------------------------------------------------------- function computeForeignPodsList (url, callback) { - // Always add a trust pod - pods_score[url] = Infinity + // Let's give 1 point to the pod we ask the friends list + pods_score[url] = 1 getForeignPodsList(url, function (foreign_pods_list) { if (foreign_pods_list.length === 0) return callback() @@ -175,16 +214,19 @@ pods_list, - function eachRequest (err, response, body, url) { + function eachRequest (err, response, body, pod, callback_each_request) { // We add the pod if it responded correctly with its public certificate if (!err && response.statusCode === 200) { - pods.add({ url: url, publicKey: body.cert }, function (err) { + pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) { if (err) { - logger.error('Error with adding %s pod.', url, { error: err }) + logger.error('Error with adding %s pod.', pod.url, { error: err }) } + + return callback_each_request() }) } else { - logger.error('Error with adding %s pod.', url, { error: err || new Error('Status not 200') }) + logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') }) + return callback_each_request() } }, diff --git a/src/utils.js b/src/utils.js index d6b26db4b..dda6c7a0a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,6 +1,7 @@ ;(function () { 'use strict' + var async = require('async') var config = require('config') var crypto = require('crypto') var fs = require('fs') @@ -30,14 +31,15 @@ } logger.debug('Sending informations to %s.', to_pod.url, { params: params }) + // Default 10 but in tests we want to be faster + var retries = utils.isTestInstance() ? 2 : 10 - // Replay 15 times, with factor 3 replay( request.post(params, function (err, response, body) { - callbackEach(err, response, body, to_pod.url) + callbackEach(err, response, body, to_pod) }), { - retries: 10, + retries: retries, factor: 3, maxTimeout: Infinity, errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] @@ -68,7 +70,13 @@ } // Make a request for each pod - for (var pod of pods) { + async.each(pods, function (pod, callback_each_async) { + function callbackEachRetryRequest (err, response, body, pod) { + callbackEach(err, response, body, pod, function () { + callback_each_async() + }) + } + var params = { url: pod.url + all_data.path, method: all_data.method @@ -93,20 +101,18 @@ key: passwordEncrypted } - makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEach) + makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest) }) })(crt, params, url, pod, signature) } else { params.json = { data: all_data.data } - makeRetryRequest(params, url, pod, signature, callbackEach) + makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) } } else { logger.debug('Make a GET/DELETE request') - makeRetryRequest(params, url, pod, signature, callbackEach) + makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) } - } - - return callback() + }, callback) } utils.certsExist = function (callback) { @@ -192,5 +198,9 @@ process.kill(-webtorrent_process.pid) } + utils.isTestInstance = function () { + return (process.env.NODE_ENV === 'test') + } + module.exports = utils })() diff --git a/src/videos.js b/src/videos.js index b95219c39..8c44cad95 100644 --- a/src/videos.js +++ b/src/videos.js @@ -78,14 +78,9 @@ data: params } - pods.makeSecureRequest(data, function (err) { - if (err) { - logger.error('Somes issues when sending this video to friends.', { error: err }) - return callback(err) - } - - return callback(null) - }) + // Do not wait the secure requests + pods.makeSecureRequest(data) + callback(null) }) }) } @@ -138,14 +133,8 @@ } // Yes this is a POST request because we add some informations in the body (signature, encrypt etc) - pods.makeSecureRequest(data, function (err) { - if (err) { - logger.error('Somes issues when sending we want to remove the video to friends.', { error: err }) - return callback(err) - } - - callback(null) - }) + pods.makeSecureRequest(data) + callback(null) }) }) }) diff --git a/test/api/friendsAdvanced.js b/test/api/friendsAdvanced.js new file mode 100644 index 000000000..ccddac4dc --- /dev/null +++ b/test/api/friendsAdvanced.js @@ -0,0 +1,154 @@ +;(function () { + 'use strict' + + var request = require('supertest') + var chai = require('chai') + var expect = chai.expect + + var utils = require('../utils') + + describe('Test advanced friends', function () { + var path = '/api/v1/pods/makefriends' + var apps = [] + var urls = [] + + function makeFriend (pod_number, callback) { + // The first pod make friend with the third + request(urls[pod_number - 1]) + .get(path) + .set('Accept', 'application/json') + .expect(204) + .end(function (err, res) { + if (err) throw err + + // Wait for the request between pods + setTimeout(function () { + callback() + }, 1000) + }) + } + + function getFriendsList (pod_number, end) { + var path = '/api/v1/pods/' + + request(urls[pod_number - 1]) + .get(path) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) + } + + function uploadVideo (pod_number, callback) { + var path = '/api/v1/videos' + + request(urls[pod_number - 1]) + .post(path) + .set('Accept', 'application/json') + .field('name', 'my super video') + .field('description', 'my super description') + .attach('input_video', __dirname + '/../fixtures/video_short.webm') + .expect(201) + .end(function (err) { + if (err) throw err + + // Wait for the retry requests + setTimeout(callback, 10000) + }) + } + + beforeEach(function (done) { + this.timeout(30000) + utils.runMultipleServers(6, function (apps_run, urls_run) { + apps = apps_run + urls = urls_run + done() + }) + }) + + afterEach(function (done) { + apps.forEach(function (app) { + process.kill(-app.pid) + }) + + if (this.ok) { + utils.flushTests(function () { + done() + }) + } else { + done() + } + }) + + it('Should make friends with two pod each in a different group', function (done) { + this.timeout(10000) + + // Pod 3 makes friend with the first one + makeFriend(3, function () { + // Pod 4 makes friend with the second one + makeFriend(4, function () { + // Now if the fifth wants to make friends with the third et the first + makeFriend(5, function () { + // It should have 0 friends + getFriendsList(5, function (err, res) { + if (err) throw err + + expect(res.body.length).to.equal(0) + + done() + }) + }) + }) + }) + }) + + it('Should make friends with the pods 1, 2, 3', function (done) { + this.timeout(100000) + + // Pods 1, 2, 3 and 4 become friends + makeFriend(2, function () { + makeFriend(1, function () { + makeFriend(4, function () { + // Kill the server 4 + apps[3].kill() + + // Expulse pod 4 from pod 1 and 2 + uploadVideo(1, function () { + uploadVideo(1, function () { + uploadVideo(2, function () { + uploadVideo(2, function () { + // Rerun server 4 + utils.runServer(4, function (app, url) { + apps[3] = app + getFriendsList(4, function (err, res) { + if (err) throw err + // Pod 4 didn't know pod 1 and 2 removed it + expect(res.body.length).to.equal(3) + + // Pod 6 ask pod 1, 2 and 3 + makeFriend(6, function () { + getFriendsList(6, function (err, res) { + if (err) throw err + + // Pod 4 should not be our friend + var result = res.body + expect(result.length).to.equal(3) + for (var pod of result) { + expect(pod.url).not.equal(urls[3]) + } + + done() + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) +})() diff --git a/test/api/friends.js b/test/api/friendsBasic.js similarity index 98% rename from test/api/friends.js rename to test/api/friendsBasic.js index 845ccd1a8..40ed34199 100644 --- a/test/api/friends.js +++ b/test/api/friendsBasic.js @@ -19,7 +19,7 @@ .end(end) } - describe('Test friends', function () { + describe('Test basic friends', function () { var apps = [] var urls = [] diff --git a/test/utils.js b/test/utils.js index 69f43d731..af3e8665d 100644 --- a/test/utils.js +++ b/test/utils.js @@ -74,8 +74,8 @@ } module.exports = { + flushTests: flushTests, runMultipleServers: runMultipleServers, - runServer: runServer, - flushTests: flushTests + runServer: runServer } })() -- 2.25.1