From feb4bdfd9b46e87aadfa7c0d5338cde887d1f58c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 11 Dec 2016 21:50:51 +0100 Subject: [PATCH] First version with PostgreSQL --- .../friend-list/friend-list.component.html | 2 +- .../app/admin/friends/shared/friend.model.ts | 2 +- .../request-stats.component.html | 2 +- .../request-stats/request-stats.component.ts | 2 +- .../requests/shared/request-stats.model.ts | 6 +- .../users/user-list/user-list.component.html | 2 +- client/src/app/shared/auth/auth-user.model.ts | 9 +- .../app/shared/search/search-field.type.ts | 2 +- .../src/app/shared/search/search.component.ts | 4 +- client/src/app/shared/users/user.model.ts | 10 +- .../src/app/videos/shared/sort-field.type.ts | 2 +- client/src/app/videos/shared/video.model.ts | 6 +- .../videos/video-list/video-list.component.ts | 2 +- .../video-list/video-miniature.component.html | 2 +- .../video-list/video-miniature.component.scss | 2 +- .../videos/video-list/video-sort.component.ts | 4 +- .../video-watch/video-watch.component.html | 2 +- config/default.yaml | 4 +- config/production.yaml.example | 2 +- config/test-1.yaml | 2 +- config/test-2.yaml | 2 +- config/test-3.yaml | 2 +- config/test-4.yaml | 2 +- config/test-5.yaml | 2 +- config/test-6.yaml | 2 +- config/test.yaml | 2 +- package.json | 4 +- scripts/clean/server/test.sh | 3 +- server.js | 9 +- server/controllers/api/clients.js | 8 +- server/controllers/api/pods.js | 15 +- server/controllers/api/remote.js | 95 ++++- server/controllers/api/requests.js | 10 +- server/controllers/api/users.js | 28 +- server/controllers/api/videos.js | 63 ++- server/controllers/client.js | 11 +- server/helpers/custom-validators/videos.js | 19 +- server/helpers/logger.js | 6 +- server/initializers/checker.js | 8 +- server/initializers/constants.js | 19 +- server/initializers/database.js | 58 +-- server/initializers/installer.js | 29 +- server/initializers/migrator.js | 12 +- server/lib/friends.js | 73 ++-- server/lib/oauth-model.js | 32 +- server/middlewares/pods.js | 1 - server/middlewares/secure.js | 6 +- server/middlewares/sort.js | 4 +- server/middlewares/validators/users.js | 13 +- server/middlewares/validators/videos.js | 17 +- server/models/application.js | 45 +- server/models/author.js | 28 ++ server/models/oauth-client.js | 72 +++- server/models/oauth-token.js | 109 +++-- server/models/pods.js | 156 ++++--- server/models/request.js | 187 +++++---- server/models/requestToPod.js | 30 ++ server/models/user.js | 132 +++--- server/models/utils.js | 31 +- server/models/video.js | 388 ++++++++++++------ server/tests/api/check-params.js | 6 +- server/tests/api/friends-basic.js | 4 +- server/tests/api/multiple-pods.js | 8 +- server/tests/api/requests.js | 29 +- server/tests/api/single-pod.js | 16 +- server/tests/api/users.js | 8 +- server/tests/utils/servers.js | 4 +- server/tests/utils/videos.js | 2 +- 68 files changed, 1160 insertions(+), 719 deletions(-) create mode 100644 server/models/author.js create mode 100644 server/models/requestToPod.js diff --git a/client/src/app/admin/friends/friend-list/friend-list.component.html b/client/src/app/admin/friends/friend-list/friend-list.component.html index 4236fc5f6..06258f8c8 100644 --- a/client/src/app/admin/friends/friend-list/friend-list.component.html +++ b/client/src/app/admin/friends/friend-list/friend-list.component.html @@ -15,7 +15,7 @@ {{ friend.id }} {{ friend.host }} {{ friend.score }} - {{ friend.createdDate | date: 'medium' }} + {{ friend.createdAt | date: 'medium' }} diff --git a/client/src/app/admin/friends/shared/friend.model.ts b/client/src/app/admin/friends/shared/friend.model.ts index 3c23feebc..462cc82ed 100644 --- a/client/src/app/admin/friends/shared/friend.model.ts +++ b/client/src/app/admin/friends/shared/friend.model.ts @@ -2,5 +2,5 @@ export interface Friend { id: string; host: string; score: number; - createdDate: Date; + createdAt: Date; } diff --git a/client/src/app/admin/requests/request-stats/request-stats.component.html b/client/src/app/admin/requests/request-stats/request-stats.component.html index b5ac59a9a..6698eac48 100644 --- a/client/src/app/admin/requests/request-stats/request-stats.component.html +++ b/client/src/app/admin/requests/request-stats/request-stats.component.html @@ -18,6 +18,6 @@
Remaining requests: - {{ stats.requests.length }} + {{ stats.totalRequests }}
diff --git a/client/src/app/admin/requests/request-stats/request-stats.component.ts b/client/src/app/admin/requests/request-stats/request-stats.component.ts index d20b12199..9e2af219c 100644 --- a/client/src/app/admin/requests/request-stats/request-stats.component.ts +++ b/client/src/app/admin/requests/request-stats/request-stats.component.ts @@ -19,7 +19,7 @@ export class RequestStatsComponent implements OnInit, OnDestroy { } ngOnDestroy() { - if (this.stats.secondsInterval !== null) { + if (this.stats !== null && this.stats.secondsInterval !== null) { clearInterval(this.interval); } } diff --git a/client/src/app/admin/requests/shared/request-stats.model.ts b/client/src/app/admin/requests/shared/request-stats.model.ts index 766e80836..49ecbc79e 100644 --- a/client/src/app/admin/requests/shared/request-stats.model.ts +++ b/client/src/app/admin/requests/shared/request-stats.model.ts @@ -7,18 +7,18 @@ export class RequestStats { maxRequestsInParallel: number; milliSecondsInterval: number; remainingMilliSeconds: number; - requests: Request[]; + totalRequests: number; constructor(hash: { maxRequestsInParallel: number, milliSecondsInterval: number, remainingMilliSeconds: number, - requests: Request[]; + totalRequests: number; }) { this.maxRequestsInParallel = hash.maxRequestsInParallel; this.milliSecondsInterval = hash.milliSecondsInterval; this.remainingMilliSeconds = hash.remainingMilliSeconds; - this.requests = hash.requests; + this.totalRequests = hash.totalRequests; } get remainingSeconds() { diff --git a/client/src/app/admin/users/user-list/user-list.component.html b/client/src/app/admin/users/user-list/user-list.component.html index 328b1be77..36193d119 100644 --- a/client/src/app/admin/users/user-list/user-list.component.html +++ b/client/src/app/admin/users/user-list/user-list.component.html @@ -14,7 +14,7 @@ {{ user.id }} {{ user.username }} - {{ user.createdDate | date: 'medium' }} + {{ user.createdAt | date: 'medium' }} diff --git a/client/src/app/shared/auth/auth-user.model.ts b/client/src/app/shared/auth/auth-user.model.ts index bdd5ea5a9..f560351f4 100644 --- a/client/src/app/shared/auth/auth-user.model.ts +++ b/client/src/app/shared/auth/auth-user.model.ts @@ -7,9 +7,6 @@ export class AuthUser extends User { USERNAME: 'username' }; - id: string; - role: string; - username: string; tokens: Tokens; static load() { @@ -17,7 +14,7 @@ export class AuthUser extends User { if (usernameLocalStorage) { return new AuthUser( { - id: localStorage.getItem(this.KEYS.ID), + id: parseInt(localStorage.getItem(this.KEYS.ID)), username: localStorage.getItem(this.KEYS.USERNAME), role: localStorage.getItem(this.KEYS.ROLE) }, @@ -35,7 +32,7 @@ export class AuthUser extends User { Tokens.flush(); } - constructor(userHash: { id: string, username: string, role: string }, hashTokens: any) { + constructor(userHash: { id: number, username: string, role: string }, hashTokens: any) { super(userHash); this.tokens = new Tokens(hashTokens); } @@ -58,7 +55,7 @@ export class AuthUser extends User { } save() { - localStorage.setItem(AuthUser.KEYS.ID, this.id); + localStorage.setItem(AuthUser.KEYS.ID, this.id.toString()); localStorage.setItem(AuthUser.KEYS.USERNAME, this.username); localStorage.setItem(AuthUser.KEYS.ROLE, this.role); this.tokens.save(); diff --git a/client/src/app/shared/search/search-field.type.ts b/client/src/app/shared/search/search-field.type.ts index 5228ee68a..6be584ed1 100644 --- a/client/src/app/shared/search/search-field.type.ts +++ b/client/src/app/shared/search/search-field.type.ts @@ -1 +1 @@ -export type SearchField = "name" | "author" | "podUrl" | "magnetUri" | "tags"; +export type SearchField = "name" | "author" | "host" | "magnetUri" | "tags"; diff --git a/client/src/app/shared/search/search.component.ts b/client/src/app/shared/search/search.component.ts index b6237469b..9f7e156ec 100644 --- a/client/src/app/shared/search/search.component.ts +++ b/client/src/app/shared/search/search.component.ts @@ -14,8 +14,8 @@ export class SearchComponent implements OnInit { fieldChoices = { name: 'Name', author: 'Author', - podUrl: 'Pod Url', - magnetUri: 'Magnet Uri', + host: 'Pod Host', + magnetUri: 'Magnet URI', tags: 'Tags' }; searchCriterias: Search = { diff --git a/client/src/app/shared/users/user.model.ts b/client/src/app/shared/users/user.model.ts index 726495d11..52d89e004 100644 --- a/client/src/app/shared/users/user.model.ts +++ b/client/src/app/shared/users/user.model.ts @@ -1,16 +1,16 @@ export class User { - id: string; + id: number; username: string; role: string; - createdDate: Date; + createdAt: Date; - constructor(hash: { id: string, username: string, role: string, createdDate?: Date }) { + constructor(hash: { id: number, username: string, role: string, createdAt?: Date }) { this.id = hash.id; this.username = hash.username; this.role = hash.role; - if (hash.createdDate) { - this.createdDate = hash.createdDate; + if (hash.createdAt) { + this.createdAt = hash.createdAt; } } diff --git a/client/src/app/videos/shared/sort-field.type.ts b/client/src/app/videos/shared/sort-field.type.ts index 6e8cc7936..74908e344 100644 --- a/client/src/app/videos/shared/sort-field.type.ts +++ b/client/src/app/videos/shared/sort-field.type.ts @@ -1,3 +1,3 @@ export type SortField = "name" | "-name" | "duration" | "-duration" - | "createdDate" | "-createdDate"; + | "createdAt" | "-createdAt"; diff --git a/client/src/app/videos/shared/video.model.ts b/client/src/app/videos/shared/video.model.ts index b51a0e9de..fae001d78 100644 --- a/client/src/app/videos/shared/video.model.ts +++ b/client/src/app/videos/shared/video.model.ts @@ -1,7 +1,7 @@ export class Video { author: string; by: string; - createdDate: Date; + createdAt: Date; description: string; duration: string; id: string; @@ -27,7 +27,7 @@ export class Video { constructor(hash: { author: string, - createdDate: string, + createdAt: string, description: string, duration: number; id: string, @@ -39,7 +39,7 @@ export class Video { thumbnailPath: string }) { this.author = hash.author; - this.createdDate = new Date(hash.createdDate); + this.createdAt = new Date(hash.createdAt); this.description = hash.description; this.duration = Video.createDurationString(hash.duration); this.id = hash.id; diff --git a/client/src/app/videos/video-list/video-list.component.ts b/client/src/app/videos/video-list/video-list.component.ts index a8b92480b..6c42ba5be 100644 --- a/client/src/app/videos/video-list/video-list.component.ts +++ b/client/src/app/videos/video-list/video-list.component.ts @@ -145,7 +145,7 @@ export class VideoListComponent implements OnInit, OnDestroy { }; } - this.sort = routeParams['sort'] || '-createdDate'; + this.sort = routeParams['sort'] || '-createdAt'; if (routeParams['page'] !== undefined) { this.pagination.currentPage = parseInt(routeParams['page']); diff --git a/client/src/app/videos/video-list/video-miniature.component.html b/client/src/app/videos/video-list/video-miniature.component.html index 16513902b..f2f4a53a9 100644 --- a/client/src/app/videos/video-list/video-miniature.component.html +++ b/client/src/app/videos/video-list/video-miniature.component.html @@ -23,6 +23,6 @@ {{ video.by }} - {{ video.createdDate | date:'short' }} + {{ video.createdAt | date:'short' }} diff --git a/client/src/app/videos/video-list/video-miniature.component.scss b/client/src/app/videos/video-list/video-miniature.component.scss index 6b3fa3bf0..d70b1b50d 100644 --- a/client/src/app/videos/video-list/video-miniature.component.scss +++ b/client/src/app/videos/video-list/video-miniature.component.scss @@ -79,7 +79,7 @@ } } - .video-miniature-author, .video-miniature-created-date { + .video-miniature-author, .video-miniature-created-at { display: block; margin-left: 1px; font-size: 12px; diff --git a/client/src/app/videos/video-list/video-sort.component.ts b/client/src/app/videos/video-list/video-sort.component.ts index ca94b07c2..53951deb4 100644 --- a/client/src/app/videos/video-list/video-sort.component.ts +++ b/client/src/app/videos/video-list/video-sort.component.ts @@ -17,8 +17,8 @@ export class VideoSortComponent { '-name': 'Name - Desc', 'duration': 'Duration - Asc', '-duration': 'Duration - Desc', - 'createdDate': 'Created Date - Asc', - '-createdDate': 'Created Date - Desc' + 'createdAt': 'Created Date - Asc', + '-createdAt': 'Created Date - Desc' }; get choiceKeys() { diff --git a/client/src/app/videos/video-watch/video-watch.component.html b/client/src/app/videos/video-watch/video-watch.component.html index 0f0fa68cc..a726ef3ff 100644 --- a/client/src/app/videos/video-watch/video-watch.component.html +++ b/client/src/app/videos/video-watch/video-watch.component.html @@ -47,7 +47,7 @@ {{ video.by }} - on {{ video.createdDate | date:'short' }} + on {{ video.createdAt | date:'short' }} diff --git a/config/default.yaml b/config/default.yaml index 90f4b9466..631400f7d 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -8,8 +8,8 @@ webserver: database: hostname: 'localhost' - port: 27017 - suffix: '-dev' + port: 5432 + suffix: '_dev' # From the project root directory storage: diff --git a/config/production.yaml.example b/config/production.yaml.example index 056ace434..743f972de 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -5,4 +5,4 @@ webserver: port: 80 database: - suffix: '-prod' + suffix: '_prod' diff --git a/config/test-1.yaml b/config/test-1.yaml index 6dcc7f294..b2a0a5422 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -6,7 +6,7 @@ webserver: port: 9001 database: - suffix: '-test1' + suffix: '_test1' # From the project root directory storage: diff --git a/config/test-2.yaml b/config/test-2.yaml index 209525963..7285f3394 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -6,7 +6,7 @@ webserver: port: 9002 database: - suffix: '-test2' + suffix: '_test2' # From the project root directory storage: diff --git a/config/test-3.yaml b/config/test-3.yaml index 15719d107..138a2cd53 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -6,7 +6,7 @@ webserver: port: 9003 database: - suffix: '-test3' + suffix: '_test3' # From the project root directory storage: diff --git a/config/test-4.yaml b/config/test-4.yaml index e6f34d013..7425f4af7 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -6,7 +6,7 @@ webserver: port: 9004 database: - suffix: '-test4' + suffix: '_test4' # From the project root directory storage: diff --git a/config/test-5.yaml b/config/test-5.yaml index fdeec76d4..1bf0de658 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -6,7 +6,7 @@ webserver: port: 9005 database: - suffix: '-test5' + suffix: '_test5' # From the project root directory storage: diff --git a/config/test-6.yaml b/config/test-6.yaml index 0c9630c89..7303a08fc 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -6,7 +6,7 @@ webserver: port: 9006 database: - suffix: '-test6' + suffix: '_test6' # From the project root directory storage: diff --git a/config/test.yaml b/config/test.yaml index 06705a987..493a23076 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -6,4 +6,4 @@ webserver: database: hostname: 'localhost' - port: 27017 + port: 5432 diff --git a/package.json b/package.json index 300af4867..bff21082f 100644 --- a/package.json +++ b/package.json @@ -56,16 +56,18 @@ "lodash": "^4.11.1", "magnet-uri": "^5.1.4", "mkdirp": "^0.5.1", - "mongoose": "^4.0.5", "morgan": "^1.5.3", "multer": "^1.1.0", "openssl-wrapper": "^0.3.4", "parse-torrent": "^5.8.0", "password-generator": "^2.0.2", + "pg": "^6.1.0", + "pg-hstore": "^2.3.2", "request": "^2.57.0", "request-replay": "^1.0.2", "rimraf": "^2.5.4", "scripty": "^1.5.0", + "sequelize": "^3.27.0", "ursa": "^0.9.1", "winston": "^2.1.1", "ws": "^1.1.1" diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 927671dd4..35d3ad50f 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh for i in $(seq 1 6); do - printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo + dropdb "peertube_test$i" rm -rf "./test$i" + createdb "peertube_test$i" done diff --git a/server.js b/server.js index 6eb022000..e54ffe69f 100644 --- a/server.js +++ b/server.js @@ -17,10 +17,9 @@ const app = express() // ----------- Database ----------- const constants = require('./server/initializers/constants') -const database = require('./server/initializers/database') const logger = require('./server/helpers/logger') - -database.connect() +// Initialize database and models +const db = require('./server/initializers/database') // ----------- Checker ----------- const checker = require('./server/initializers/checker') @@ -39,9 +38,7 @@ if (errorMessage !== null) { const customValidators = require('./server/helpers/custom-validators') const installer = require('./server/initializers/installer') const migrator = require('./server/initializers/migrator') -const mongoose = require('mongoose') const routes = require('./server/controllers') -const Request = mongoose.model('Request') // ----------- Command line ----------- @@ -130,7 +127,7 @@ installer.installApplication(function (err) { // ----------- Make the server listening ----------- server.listen(port, function () { // Activate the pool requests - Request.activate() + db.Request.activate() logger.info('Server listening on port %d', port) logger.info('Webserver: %s', constants.CONFIG.WEBSERVER.URL) diff --git a/server/controllers/api/clients.js b/server/controllers/api/clients.js index 7755f6c2b..cf83cb835 100644 --- a/server/controllers/api/clients.js +++ b/server/controllers/api/clients.js @@ -1,13 +1,11 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Client = mongoose.model('OAuthClient') - const router = express.Router() router.get('/local', getLocalClient) @@ -27,12 +25,12 @@ function getLocalClient (req, res, next) { return res.type('json').status(403).end() } - Client.loadFirstClient(function (err, client) { + db.OAuthClient.loadFirstClient(function (err, client) { if (err) return next(err) if (!client) return next(new Error('No client available.')) res.json({ - client_id: client._id, + client_id: client.clientId, client_secret: client.clientSecret }) }) diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index 7857fcee0..79f3f9d8d 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -1,9 +1,9 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') @@ -15,7 +15,6 @@ const validators = middlewares.validators.pods const signatureValidator = middlewares.validators.remote.signature const router = express.Router() -const Pod = mongoose.model('Pod') router.get('/', listPods) router.post('/', @@ -53,15 +52,15 @@ function addPods (req, res, next) { waterfall([ function addPod (callback) { - const pod = new Pod(informations) - pod.save(function (err, podCreated) { + const pod = db.Pod.build(informations) + pod.save().asCallback(function (err, podCreated) { // Be sure about the number of parameters for the callback return callback(err, podCreated) }) }, function sendMyVideos (podCreated, callback) { - friends.sendOwnedVideosToPod(podCreated._id) + friends.sendOwnedVideosToPod(podCreated.id) callback(null) }, @@ -84,7 +83,7 @@ function addPods (req, res, next) { } function listPods (req, res, next) { - Pod.list(function (err, podsList) { + db.Pod.list(function (err, podsList) { if (err) return next(err) res.json(getFormatedPods(podsList)) @@ -111,11 +110,11 @@ function removePods (req, res, next) { waterfall([ function loadPod (callback) { - Pod.loadByHost(host, callback) + db.Pod.loadByHost(host, callback) }, function removePod (pod, callback) { - pod.remove(callback) + pod.destroy().asCallback(callback) } ], function (err) { if (err) return next(err) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index f1046c534..d856576a9 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -3,15 +3,15 @@ const each = require('async/each') const eachSeries = require('async/eachSeries') const express = require('express') -const mongoose = require('mongoose') +const waterfall = require('async/waterfall') +const db = require('../../initializers/database') const middlewares = require('../../middlewares') const secureMiddleware = middlewares.secure const validators = middlewares.validators.remote const logger = require('../../helpers/logger') const router = express.Router() -const Video = mongoose.model('Video') router.post('/videos', validators.signature, @@ -53,34 +53,99 @@ function remoteVideos (req, res, next) { function addRemoteVideo (videoToCreateData, fromHost, callback) { logger.debug('Adding remote video "%s".', videoToCreateData.name) - const video = new Video(videoToCreateData) - video.podHost = fromHost - Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { - if (err) { - logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) - return callback(err) + waterfall([ + + function findOrCreatePod (callback) { + fromHost + + const query = { + where: { + host: fromHost + }, + defaults: { + host: fromHost + } + } + + db.Pod.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function findOrCreateAuthor (pod, callback) { + const username = videoToCreateData.author + + const query = { + where: { + name: username, + podId: pod.id + }, + defaults: { + name: username, + podId: pod.id + } + } + + db.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function createVideoObject (author, callback) { + const videoData = { + name: videoToCreateData.name, + remoteId: videoToCreateData.remoteId, + extname: videoToCreateData.extname, + infoHash: videoToCreateData.infoHash, + description: videoToCreateData.description, + authorId: author.id, + duration: videoToCreateData.duration, + tags: videoToCreateData.tags + } + + const video = db.Video.build(videoData) + + return callback(null, video) + }, + + function generateThumbnail (video, callback) { + db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { + if (err) { + logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) + return callback(err) + } + + video.save().asCallback(callback) + }) + }, + + function insertIntoDB (video, callback) { + video.save().asCallback(callback) } - video.save(callback) - }) + ], callback) } function removeRemoteVideo (videoToRemoveData, fromHost, callback) { + // TODO: use bulkDestroy? + // We need the list because we have to remove some other stuffs (thumbnail etc) - Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { + db.Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { if (err) { - logger.error('Cannot list videos from host and magnets.', { error: err }) + logger.error('Cannot list videos from host and remote id.', { error: err.message }) return callback(err) } if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { magnetUri: videoToRemoveData.magnetUri, podHost: fromHost }) + logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromHost }) } each(videosList, function (video, callbackEach) { - logger.debug('Removing remote video %s.', video.magnetUri) + logger.debug('Removing remote video %s.', video.remoteId) - video.remove(callbackEach) + video.destroy().asCallback(callbackEach) }, callback) }) } diff --git a/server/controllers/api/requests.js b/server/controllers/api/requests.js index 52aad6997..1f9193fc8 100644 --- a/server/controllers/api/requests.js +++ b/server/controllers/api/requests.js @@ -1,15 +1,13 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const middlewares = require('../../middlewares') const admin = middlewares.admin const oAuth = middlewares.oauth -const Request = mongoose.model('Request') - const router = express.Router() router.get('/stats', @@ -25,13 +23,13 @@ module.exports = router // --------------------------------------------------------------------------- function getStatsRequests (req, res, next) { - Request.list(function (err, requests) { + db.Request.countTotalRequests(function (err, totalRequests) { if (err) return next(err) return res.json({ - requests: requests, + totalRequests: totalRequests, maxRequestsInParallel: constants.REQUESTS_IN_PARALLEL, - remainingMilliSeconds: Request.remainingMilliSeconds(), + remainingMilliSeconds: db.Request.remainingMilliSeconds(), milliSecondsInterval: constants.REQUESTS_INTERVAL }) }) diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index b4d687312..890028b36 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -2,10 +2,10 @@ const each = require('async/each') const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const friends = require('../../lib/friends') const logger = require('../../helpers/logger') const middlewares = require('../../middlewares') @@ -17,9 +17,6 @@ const validatorsPagination = middlewares.validators.pagination const validatorsSort = middlewares.validators.sort const validatorsUsers = middlewares.validators.users -const User = mongoose.model('User') -const Video = mongoose.model('Video') - const router = express.Router() router.get('/me', oAuth.authenticate, getUserInformation) @@ -62,13 +59,13 @@ module.exports = router // --------------------------------------------------------------------------- function createUser (req, res, next) { - const user = new User({ + const user = db.User.build({ username: req.body.username, password: req.body.password, role: constants.USER_ROLES.USER }) - user.save(function (err, createdUser) { + user.save().asCallback(function (err, createdUser) { if (err) return next(err) return res.type('json').status(204).end() @@ -76,7 +73,7 @@ function createUser (req, res, next) { } function getUserInformation (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) return res.json(user.toFormatedJSON()) @@ -84,7 +81,7 @@ function getUserInformation (req, res, next) { } function listUsers (req, res, next) { - User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { + db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { if (err) return next(err) res.json(getFormatedUsers(usersList, usersTotal)) @@ -94,18 +91,19 @@ function listUsers (req, res, next) { function removeUser (req, res, next) { waterfall([ function getUser (callback) { - User.loadById(req.params.id, callback) + db.User.loadById(req.params.id, callback) }, + // TODO: use foreignkey? function getVideos (user, callback) { - Video.listOwnedByAuthor(user.username, function (err, videos) { + db.Video.listOwnedByAuthor(user.username, function (err, videos) { return callback(err, user, videos) }) }, function removeVideosFromDB (user, videos, callback) { each(videos, function (video, callbackEach) { - video.remove(callbackEach) + video.destroy().asCallback(callbackEach) }, function (err) { return callback(err, user, videos) }) @@ -115,7 +113,7 @@ function removeUser (req, res, next) { videos.forEach(function (video) { const params = { name: video.name, - magnetUri: video.magnetUri + remoteId: video.id } friends.removeVideoToFriends(params) @@ -125,7 +123,7 @@ function removeUser (req, res, next) { }, function removeUserFromDB (user, callback) { - user.remove(callback) + user.destroy().asCallback(callback) } ], function andFinally (err) { if (err) { @@ -138,11 +136,11 @@ function removeUser (req, res, next) { } function updateUser (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) user.password = req.body.password - user.save(function (err) { + user.save().asCallback(function (err) { if (err) return next(err) return res.sendStatus(204) diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index daf452573..a61f2b2c9 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -2,12 +2,12 @@ const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const multer = require('multer') const path = require('path') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') @@ -22,7 +22,6 @@ const sort = middlewares.sort const utils = require('../../helpers/utils') const router = express.Router() -const Video = mongoose.model('Video') // multer configuration const storage = multer.diskStorage({ @@ -87,40 +86,60 @@ function addVideo (req, res, next) { const videoInfos = req.body waterfall([ - function createVideoObject (callback) { - const id = mongoose.Types.ObjectId() + function findOrCreateAuthor (callback) { + const username = res.locals.oauth.token.user.username + + const query = { + where: { + name: username, + podId: null + }, + defaults: { + name: username, + podId: null // null because it is OUR pod + } + } + + db.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function createVideoObject (author, callback) { const videoData = { - _id: id, name: videoInfos.name, remoteId: null, extname: path.extname(videoFile.filename), description: videoInfos.description, - author: res.locals.oauth.token.user.username, duration: videoFile.duration, - tags: videoInfos.tags + tags: videoInfos.tags, + authorId: author.id } - const video = new Video(videoData) + const video = db.Video.build(videoData) - return callback(null, video) + return callback(null, author, video) }, - // Set the videoname the same as the MongoDB id - function renameVideoFile (video, callback) { + // Set the videoname the same as the id + function renameVideoFile (author, video, callback) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, video) + return callback(err, author, video) }) }, - function insertIntoDB (video, callback) { - video.save(function (err, video) { - // Assert there are only one argument sent to the next function (video) - return callback(err, video) + function insertIntoDB (author, video, callback) { + video.save().asCallback(function (err, videoCreated) { + // Do not forget to add Author informations to the created video + videoCreated.Author = author + + return callback(err, videoCreated) }) }, @@ -147,7 +166,7 @@ function addVideo (req, res, next) { } function getVideo (req, res, next) { - Video.load(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthorAndPod(req.params.id, function (err, video) { if (err) return next(err) if (!video) { @@ -159,7 +178,7 @@ function getVideo (req, res, next) { } function listVideos (req, res, next) { - Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { + db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) res.json(getFormatedVideos(videosList, videosTotal)) @@ -171,11 +190,11 @@ function removeVideo (req, res, next) { waterfall([ function getVideo (callback) { - Video.load(videoId, callback) + db.Video.load(videoId, callback) }, function removeFromDB (video, callback) { - video.remove(function (err) { + video.destroy().asCallback(function (err) { if (err) return callback(err) return callback(null, video) @@ -185,7 +204,7 @@ function removeVideo (req, res, next) { function sendInformationToFriends (video, callback) { const params = { name: video.name, - remoteId: video._id + remoteId: video.id } friends.removeVideoToFriends(params) @@ -203,7 +222,7 @@ function removeVideo (req, res, next) { } function searchVideos (req, res, next) { - Video.search(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + db.Video.searchAndPopulateAuthorAndPod(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) diff --git a/server/controllers/client.js b/server/controllers/client.js index 572db6133..a5fac5626 100644 --- a/server/controllers/client.js +++ b/server/controllers/client.js @@ -3,13 +3,12 @@ const parallel = require('async/parallel') const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const path = require('path') const validator = require('express-validator').validator const constants = require('../initializers/constants') +const db = require('../initializers/database') -const Video = mongoose.model('Video') const router = express.Router() const opengraphComment = '' @@ -45,14 +44,14 @@ function addOpenGraphTags (htmlStringPage, video) { if (video.isOwned()) { basePreviewUrlHttp = constants.CONFIG.WEBSERVER.URL } else { - basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.podHost + basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.Author.Pod.host } // We fetch the remote preview (bigger than the thumbnail) // This should not overhead the remote server since social websites put in a cache the OpenGraph tags // We can't use the thumbnail because these social websites want bigger images (> 200x200 for Facebook for example) const previewUrl = basePreviewUrlHttp + constants.STATIC_PATHS.PREVIEWS + video.getPreviewName() - const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video._id + const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video.id const metaTags = { 'og:type': 'video', @@ -86,7 +85,7 @@ function generateWatchHtmlPage (req, res, next) { const videoId = req.params.id // Let Angular application handle errors - if (!validator.isMongoId(videoId)) return res.sendFile(indexPath) + if (!validator.isUUID(videoId, 4)) return res.sendFile(indexPath) parallel({ file: function (callback) { @@ -94,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) { }, video: function (callback) { - Video.load(videoId, callback) + db.Video.loadAndPopulateAuthorAndPod(videoId, callback) } }, function (err, results) { if (err) return next(err) diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 1a7753265..be8256a80 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -13,7 +13,7 @@ const videosValidators = { isVideoDateValid, isVideoDescriptionValid, isVideoDurationValid, - isVideoMagnetValid, + isVideoInfoHashValid, isVideoNameValid, isVideoPodHostValid, isVideoTagsValid, @@ -28,14 +28,15 @@ function isEachRemoteVideosValid (requests) { return ( isRequestTypeAddValid(request.type) && isVideoAuthorValid(video.author) && - isVideoDateValid(video.createdDate) && + isVideoDateValid(video.createdAt) && isVideoDescriptionValid(video.description) && isVideoDurationValid(video.duration) && - isVideoMagnetValid(video.magnet) && + isVideoInfoHashValid(video.infoHash) && isVideoNameValid(video.name) && isVideoTagsValid(video.tags) && isVideoThumbnail64Valid(video.thumbnailBase64) && - isVideoRemoteIdValid(video.remoteId) + isVideoRemoteIdValid(video.remoteId) && + isVideoExtnameValid(video.extname) ) || ( isRequestTypeRemoveValid(request.type) && @@ -61,8 +62,12 @@ function isVideoDurationValid (value) { return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) } -function isVideoMagnetValid (value) { - return validator.isLength(value.infoHash, VIDEOS_CONSTRAINTS_FIELDS.MAGNET.INFO_HASH) +function isVideoExtnameValid (value) { + return VIDEOS_CONSTRAINTS_FIELDS.EXTNAME.indexOf(value) !== -1 +} + +function isVideoInfoHashValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) } function isVideoNameValid (value) { @@ -93,7 +98,7 @@ function isVideoThumbnail64Valid (value) { } function isVideoRemoteIdValid (value) { - return validator.isMongoId(value) + return validator.isUUID(value, 4) } // --------------------------------------------------------------------------- diff --git a/server/helpers/logger.js b/server/helpers/logger.js index fcc1789fd..281acedb8 100644 --- a/server/helpers/logger.js +++ b/server/helpers/logger.js @@ -22,7 +22,8 @@ const logger = new winston.Logger({ json: true, maxsize: 5242880, maxFiles: 5, - colorize: false + colorize: false, + prettyPrint: true }), new winston.transports.Console({ level: 'debug', @@ -30,7 +31,8 @@ const logger = new winston.Logger({ handleExceptions: true, humanReadableUnhandledException: true, json: false, - colorize: true + colorize: true, + prettyPrint: true }) ], exitOnError: true diff --git a/server/initializers/checker.js b/server/initializers/checker.js index aea013fa9..7b402de82 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -1,10 +1,8 @@ 'use strict' const config = require('config') -const mongoose = require('mongoose') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') +const db = require('./database') const checker = { checkConfig, @@ -44,7 +42,7 @@ function checkMissedConfig () { } function clientsExist (callback) { - Client.list(function (err, clients) { + db.OAuthClient.list(function (err, clients) { if (err) return callback(err) return callback(null, clients.length !== 0) @@ -52,7 +50,7 @@ function clientsExist (callback) { } function usersExist (callback) { - User.countTotal(function (err, totalUsers) { + db.User.countTotal(function (err, totalUsers) { if (err) return callback(err) return callback(null, totalUsers !== 0) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 3ddf87454..1ad0c82a0 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -14,13 +14,13 @@ const PAGINATION_COUNT_DEFAULT = 15 // Sortable columns per schema const SEARCHABLE_COLUMNS = { - VIDEOS: [ 'name', 'magnetUri', 'podHost', 'author', 'tags' ] + VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ] } // Sortable columns per schema const SORTABLE_COLUMNS = { - USERS: [ 'username', '-username', 'createdDate', '-createdDate' ], - VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ] + USERS: [ 'username', '-username', 'createdAt', '-createdAt' ], + VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] } const OAUTH_LIFETIME = { @@ -67,9 +67,8 @@ const CONSTRAINTS_FIELDS = { VIDEOS: { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length - MAGNET: { - INFO_HASH: { min: 10, max: 50 } // Length - }, + EXTNAME: [ '.mp4', '.ogv', '.webm' ], + INFO_HASH: { min: 10, max: 50 }, // Length DURATION: { min: 1, max: 7200 }, // Number TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length @@ -88,7 +87,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MONGO_MIGRATION_SCRIPTS = [ +const MIGRATION_SCRIPTS = [ { script: '0005-create-application', version: 5 @@ -122,7 +121,7 @@ const MONGO_MIGRATION_SCRIPTS = [ version: 40 } ] -const LAST_MONGO_SCHEMA_VERSION = (maxBy(MONGO_MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version'] // --------------------------------------------------------------------------- @@ -198,8 +197,8 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_MONGO_SCHEMA_VERSION, - MONGO_MIGRATION_SCRIPTS, + LAST_SQL_SCHEMA_VERSION, + MIGRATION_SCRIPTS, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, diff --git a/server/initializers/database.js b/server/initializers/database.js index 0564e4e77..cc6f59b63 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -1,36 +1,46 @@ 'use strict' -const mongoose = require('mongoose') +const fs = require('fs') +const path = require('path') +const Sequelize = require('sequelize') const constants = require('../initializers/constants') const logger = require('../helpers/logger') -// Bootstrap models -require('../models/application') -require('../models/oauth-token') -require('../models/user') -require('../models/oauth-client') -require('../models/video') -// Request model needs Video model -require('../models/pods') -// Request model needs Pod model -require('../models/request') - -const database = { - connect: connect -} - -function connect () { - mongoose.Promise = global.Promise - mongoose.connect('mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME) - mongoose.connection.on('error', function () { - throw new Error('Mongodb connection error.') +const database = {} + +const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { + dialect: 'postgres', + host: constants.CONFIG.DATABASE.HOSTNAME, + port: constants.CONFIG.DATABASE.PORT +}) + +const modelDirectory = path.join(__dirname, '..', 'models') +fs.readdir(modelDirectory, function (err, files) { + if (err) throw err + + files.filter(function (file) { + if (file === 'utils.js') return false + + return true }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) - mongoose.connection.on('open', function () { - logger.info('Connected to mongodb.') + database[model.name] = model }) -} + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + logger.info('Database is ready.') +}) + +database.sequelize = sequelize +database.Sequelize = Sequelize // --------------------------------------------------------------------------- diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 1df300ba8..4823bc8c8 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -3,26 +3,27 @@ const config = require('config') const each = require('async/each') const mkdirp = require('mkdirp') -const mongoose = require('mongoose') const passwordGenerator = require('password-generator') const path = require('path') const series = require('async/series') const checker = require('./checker') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') const peertubeCrypto = require('../helpers/peertube-crypto') -const Application = mongoose.model('Application') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') - const installer = { installApplication } function installApplication (callback) { series([ + function createDatabase (callbackAsync) { + db.sequelize.sync().asCallback(callbackAsync) + // db.sequelize.sync({ force: true }).asCallback(callbackAsync) + }, + function createDirectories (callbackAsync) { createDirectoriesIfNotExist(callbackAsync) }, @@ -65,16 +66,18 @@ function createOAuthClientIfNotExist (callback) { logger.info('Creating a default OAuth Client.') - const secret = passwordGenerator(32, false) - const client = new Client({ + const id = passwordGenerator(32, false, /[a-z0-9]/) + const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) + const client = db.OAuthClient.build({ + clientId: id, clientSecret: secret, grants: [ 'password', 'refresh_token' ] }) - client.save(function (err, createdClient) { + client.save().asCallback(function (err, createdClient) { if (err) return callback(err) - logger.info('Client id: ' + createdClient._id) + logger.info('Client id: ' + createdClient.clientId) logger.info('Client secret: ' + createdClient.clientSecret) return callback(null) @@ -106,21 +109,21 @@ function createOAuthAdminIfNotExist (callback) { password = passwordGenerator(8, true) } - const user = new User({ + const user = db.User.build({ username, password, role }) - user.save(function (err, createdUser) { + user.save().asCallback(function (err, createdUser) { if (err) return callback(err) logger.info('Username: ' + username) logger.info('User password: ' + password) logger.info('Creating Application collection.') - const application = new Application({ mongoSchemaVersion: constants.LAST_MONGO_SCHEMA_VERSION }) - application.save(callback) + const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION }) + application.save().asCallback(callback) }) }) } diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 6b31d994f..9e5350e60 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,24 +1,22 @@ 'use strict' const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') const path = require('path') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') -const Application = mongoose.model('Application') - const migrator = { migrate: migrate } function migrate (callback) { - Application.loadMongoSchemaVersion(function (err, actualVersion) { + db.Application.loadSqlSchemaVersion(function (err, actualVersion) { if (err) return callback(err) // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) { + if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) { logger.info('Begin migrations.') eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { @@ -36,12 +34,12 @@ function migrate (callback) { if (err) return callbackEach(err) // Update the new mongo version schema - Application.updateMongoSchemaVersion(versionScript, callbackEach) + db.Application.updateSqlSchemaVersion(versionScript, callbackEach) }) }, function (err) { if (err) return callback(err) - logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION) + logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION) return callback(null) }) } else { diff --git a/server/lib/friends.js b/server/lib/friends.js index eaea040ca..3ed29f651 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -4,18 +4,14 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') const eachSeries = require('async/eachSeries') const fs = require('fs') -const mongoose = require('mongoose') const request = require('request') const waterfall = require('async/waterfall') const constants = require('../initializers/constants') +const db = require('../initializers/database') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') -const Request = mongoose.model('Request') -const Video = mongoose.model('Video') - const friends = { addVideoToFriends, hasFriends, @@ -31,7 +27,7 @@ function addVideoToFriends (video) { } function hasFriends (callback) { - Pod.countAll(function (err, count) { + db.Pod.countAll(function (err, count) { if (err) return callback(err) const hasFriends = (count !== 0) @@ -69,13 +65,13 @@ function makeFriends (hosts, callback) { function quitFriends (callback) { // Stop pool requests - Request.deactivate() + db.Request.deactivate() // Flush pool requests - Request.flush() + db.Request.flush() waterfall([ function getPodsList (callbackAsync) { - return Pod.list(callbackAsync) + return db.Pod.list(callbackAsync) }, function announceIQuitMyFriends (pods, callbackAsync) { @@ -103,12 +99,12 @@ function quitFriends (callback) { function removePodsFromDB (pods, callbackAsync) { each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, callbackAsync) } ], function (err) { // Don't forget to re activate the scheduler, even if there was an error - Request.activate() + db.Request.activate() if (err) return callback(err) @@ -122,7 +118,7 @@ function removeVideoToFriends (videoParams) { } function sendOwnedVideosToPod (podId) { - Video.listOwned(function (err, videosList) { + db.Video.listOwnedAndPopulateAuthor(function (err, videosList) { if (err) { logger.error('Cannot get the list of videos we own.') return @@ -200,9 +196,9 @@ function getForeignPodsList (host, callback) { function makeRequestsToWinningPods (cert, podsList, callback) { // Stop pool requests - Request.deactivate() + db.Request.deactivate() // Flush pool requests - Request.forceSend() + db.Request.forceSend() eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { const params = { @@ -222,8 +218,8 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } if (res.statusCode === 200) { - const podObj = new Pod({ host: pod.host, publicKey: body.cert }) - podObj.save(function (err, podCreated) { + const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert }) + podObj.save().asCallback(function (err, podCreated) { if (err) { logger.error('Cannot add friend %s pod.', pod.host, { error: err }) return callbackEach() @@ -242,28 +238,57 @@ function makeRequestsToWinningPods (cert, podsList, callback) { }, function endRequests () { // Final callback, we've ended all the requests // Now we made new friends, we can re activate the pool of requests - Request.activate() + db.Request.activate() logger.debug('makeRequestsToWinningPods finished.') return callback() }) } +// Wrapper that populate "to" argument with all our friends if it is not specified function createRequest (type, endpoint, data, to) { - const req = new Request({ + if (to) return _createRequest(type, endpoint, data, to) + + // If the "to" pods is not specified, we send the request to all our friends + db.Pod.listAllIds(function (err, podIds) { + if (err) { + logger.error('Cannot get pod ids', { error: err }) + return + } + + return _createRequest(type, endpoint, data, podIds) + }) +} + +function _createRequest (type, endpoint, data, to) { + const pods = [] + + // If there are no destination pods abort + if (to.length === 0) return + + to.forEach(function (toPod) { + pods.push(db.Pod.build({ id: toPod })) + }) + + const createQuery = { endpoint, request: { type: type, data: data } - }) - - if (to) { - req.to = to } - req.save(function (err) { - if (err) logger.error('Cannot save the request.', { error: err }) + // We run in transaction to keep coherency between Request and RequestToPod tables + db.sequelize.transaction(function (t) { + const dbRequestOptions = { + transaction: t + } + + return db.Request.create(createQuery, dbRequestOptions).then(function (request) { + return request.setPods(pods, dbRequestOptions) + }) + }).asCallback(function (err) { + if (err) logger.error('Error in createRequest transaction.', { error: err }) }) } diff --git a/server/lib/oauth-model.js b/server/lib/oauth-model.js index d011c4b72..1c12f1b14 100644 --- a/server/lib/oauth-model.js +++ b/server/lib/oauth-model.js @@ -1,11 +1,6 @@ -const mongoose = require('mongoose') - +const db = require('../initializers/database') const logger = require('../helpers/logger') -const OAuthClient = mongoose.model('OAuthClient') -const OAuthToken = mongoose.model('OAuthToken') -const User = mongoose.model('User') - // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications const OAuthModel = { getAccessToken, @@ -21,27 +16,25 @@ const OAuthModel = { function getAccessToken (bearerToken) { logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') - return OAuthToken.getByTokenAndPopulateUser(bearerToken) + return db.OAuthToken.getByTokenAndPopulateUser(bearerToken) } function getClient (clientId, clientSecret) { logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') - // TODO req validator - const mongoId = new mongoose.mongo.ObjectID(clientId) - return OAuthClient.getByIdAndSecret(mongoId, clientSecret) + return db.OAuthClient.getByIdAndSecret(clientId, clientSecret) } function getRefreshToken (refreshToken) { logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') - return OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) + return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) } function getUser (username, password) { logger.debug('Getting User (username: ' + username + ', password: ' + password + ').') - return User.getByUsername(username).then(function (user) { + return db.User.getByUsername(username).then(function (user) { if (!user) return null // We need to return a promise @@ -60,8 +53,8 @@ function getUser (username, password) { } function revokeToken (token) { - return OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { - if (tokenDB) tokenDB.remove() + return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { + if (tokenDB) tokenDB.destroy() /* * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js @@ -80,18 +73,19 @@ function revokeToken (token) { function saveToken (token, client, user) { logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') - const tokenObj = new OAuthToken({ + const tokenToCreate = { accessToken: token.accessToken, accessTokenExpiresAt: token.accessTokenExpiresAt, - client: client.id, refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, - user: user.id - }) + oAuthClientId: client.id, + userId: user.id + } - return tokenObj.save().then(function (tokenCreated) { + return db.OAuthToken.create(tokenToCreate).then(function (tokenCreated) { tokenCreated.client = client tokenCreated.user = user + return tokenCreated }).catch(function (err) { throw err diff --git a/server/middlewares/pods.js b/server/middlewares/pods.js index 487ea1259..e38fb341d 100644 --- a/server/middlewares/pods.js +++ b/server/middlewares/pods.js @@ -44,7 +44,6 @@ module.exports = podsMiddleware function getHostWithPort (host) { const splitted = host.split(':') - console.log(splitted) // The port was not specified if (splitted.length === 1) { if (constants.REMOTE_SCHEME.HTTP === 'https') return host + ':443' diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index ee836beed..b7b4cdfb4 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -1,18 +1,16 @@ 'use strict' +const db = require('../initializers/database') const logger = require('../helpers/logger') -const mongoose = require('mongoose') const peertubeCrypto = require('../helpers/peertube-crypto') -const Pod = mongoose.model('Pod') - const secureMiddleware = { checkSignature } function checkSignature (req, res, next) { const host = req.body.signature.host - Pod.loadByHost(host, function (err, pod) { + db.Pod.loadByHost(host, function (err, pod) { if (err) { logger.error('Cannot get signed host in body.', { error: err }) return res.sendStatus(500) diff --git a/server/middlewares/sort.js b/server/middlewares/sort.js index f0b7274eb..477e10571 100644 --- a/server/middlewares/sort.js +++ b/server/middlewares/sort.js @@ -6,13 +6,13 @@ const sortMiddleware = { } function setUsersSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' return next() } function setVideosSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' return next() } diff --git a/server/middlewares/validators/users.js b/server/middlewares/validators/users.js index 02e4f34cb..0629550bc 100644 --- a/server/middlewares/validators/users.js +++ b/server/middlewares/validators/users.js @@ -1,12 +1,9 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const User = mongoose.model('User') - const validatorsUsers = { usersAdd, usersRemove, @@ -20,7 +17,7 @@ function usersAdd (req, res, next) { logger.debug('Checking usersAdd parameters', { parameters: req.body }) checkErrors(req, res, function () { - User.loadByUsername(req.body.username, function (err, user) { + db.User.loadByUsername(req.body.username, function (err, user) { if (err) { logger.error('Error in usersAdd request validator.', { error: err }) return res.sendStatus(500) @@ -34,12 +31,12 @@ function usersAdd (req, res, next) { } function usersRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() logger.debug('Checking usersRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - User.loadById(req.params.id, function (err, user) { + db.User.loadById(req.params.id, function (err, user) { if (err) { logger.error('Error in usersRemove request validator.', { error: err }) return res.sendStatus(500) @@ -55,7 +52,7 @@ function usersRemove (req, res, next) { } function usersUpdate (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() // Add old password verification req.checkBody('password', 'Should have a valid password').isUserPasswordValid() diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 76e943e77..7e90ca047 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -1,14 +1,11 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors const constants = require('../../initializers/constants') const customVideosValidators = require('../../helpers/custom-validators').videos +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Video = mongoose.model('Video') - const validatorsVideos = { videosAdd, videosGet, @@ -29,7 +26,7 @@ function videosAdd (req, res, next) { checkErrors(req, res, function () { const videoFile = req.files.videofile[0] - Video.getDurationFromFile(videoFile.path, function (err, duration) { + db.Video.getDurationFromFile(videoFile.path, function (err, duration) { if (err) { return res.status(400).send('Cannot retrieve metadata of the file.') } @@ -45,12 +42,12 @@ function videosAdd (req, res, next) { } function videosGet (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosGet parameters', { parameters: req.params }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { + db.Video.load(req.params.id, function (err, video) { if (err) { logger.error('Error in videosGet request validator.', { error: err }) return res.sendStatus(500) @@ -64,12 +61,12 @@ function videosGet (req, res, next) { } function videosRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthor(req.params.id, function (err, video) { if (err) { logger.error('Error in videosRemove request validator.', { error: err }) return res.sendStatus(500) @@ -77,7 +74,7 @@ function videosRemove (req, res, next) { if (!video) return res.status(404).send('Video not found') else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod') - else if (video.author !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') + else if (video.Author.name !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') next() }) diff --git a/server/models/application.js b/server/models/application.js index 452ac4283..ec1d7b122 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,31 +1,36 @@ -const mongoose = require('mongoose') +module.exports = function (sequelize, DataTypes) { + const Application = sequelize.define('Application', + { + sqlSchemaVersion: { + type: DataTypes.INTEGER, + defaultValue: 0 + } + }, + { + classMethods: { + loadSqlSchemaVersion, + updateSqlSchemaVersion + } + } + ) + + return Application +} // --------------------------------------------------------------------------- -const ApplicationSchema = mongoose.Schema({ - mongoSchemaVersion: { - type: Number, - default: 0 +function loadSqlSchemaVersion (callback) { + const query = { + attributes: [ 'sqlSchemaVersion' ] } -}) - -ApplicationSchema.statics = { - loadMongoSchemaVersion, - updateMongoSchemaVersion -} - -mongoose.model('Application', ApplicationSchema) - -// --------------------------------------------------------------------------- -function loadMongoSchemaVersion (callback) { - return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) { - const version = data ? data.mongoSchemaVersion : 0 + return this.findOne(query).asCallback(function (err, data) { + const version = data ? data.sqlSchemaVersion : 0 return callback(err, version) }) } -function updateMongoSchemaVersion (newVersion, callback) { - return this.update({}, { mongoSchemaVersion: newVersion }, callback) +function updateSqlSchemaVersion (newVersion, callback) { + return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback) } diff --git a/server/models/author.js b/server/models/author.js new file mode 100644 index 000000000..493c2ca11 --- /dev/null +++ b/server/models/author.js @@ -0,0 +1,28 @@ +module.exports = function (sequelize, DataTypes) { + const Author = sequelize.define('Author', + { + name: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate + } + } + ) + + return Author +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: true + }, + onDelete: 'cascade' + }) +} diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index a1aefa985..15118591a 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -1,33 +1,63 @@ -const mongoose = require('mongoose') - -// --------------------------------------------------------------------------- - -const OAuthClientSchema = mongoose.Schema({ - clientSecret: String, - grants: Array, - redirectUris: Array -}) - -OAuthClientSchema.path('clientSecret').required(true) - -OAuthClientSchema.statics = { - getByIdAndSecret, - list, - loadFirstClient +module.exports = function (sequelize, DataTypes) { + const OAuthClient = sequelize.define('OAuthClient', + { + clientId: { + type: DataTypes.STRING + }, + clientSecret: { + type: DataTypes.STRING + }, + grants: { + type: DataTypes.ARRAY(DataTypes.STRING) + }, + redirectUris: { + type: DataTypes.ARRAY(DataTypes.STRING) + } + }, + { + classMethods: { + associate, + + getByIdAndSecret, + list, + loadFirstClient + } + } + ) + + return OAuthClient } -mongoose.model('OAuthClient', OAuthClientSchema) +// TODO: validation +// OAuthClientSchema.path('clientSecret').required(true) // --------------------------------------------------------------------------- +function associate (models) { + this.hasMany(models.OAuthToken, { + foreignKey: { + name: 'oAuthClientId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function list (callback) { - return this.find(callback) + return this.findAll().asCallback(callback) } function loadFirstClient (callback) { - return this.findOne({}, callback) + return this.findOne().asCallback(callback) } -function getByIdAndSecret (id, clientSecret) { - return this.findOne({ _id: id, clientSecret: clientSecret }).exec() +function getByIdAndSecret (clientId, clientSecret) { + const query = { + where: { + clientId: clientId, + clientSecret: clientSecret + } + } + + return this.findOne(query) } diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index aff73bfb1..c9108bf95 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -1,42 +1,71 @@ -const mongoose = require('mongoose') - const logger = require('../helpers/logger') // --------------------------------------------------------------------------- -const OAuthTokenSchema = mongoose.Schema({ - accessToken: String, - accessTokenExpiresAt: Date, - client: { type: mongoose.Schema.Types.ObjectId, ref: 'OAuthClient' }, - refreshToken: String, - refreshTokenExpiresAt: Date, - user: { type: mongoose.Schema.Types.ObjectId, ref: 'User' } -}) - -OAuthTokenSchema.path('accessToken').required(true) -OAuthTokenSchema.path('client').required(true) -OAuthTokenSchema.path('user').required(true) - -OAuthTokenSchema.statics = { - getByRefreshTokenAndPopulateClient, - getByTokenAndPopulateUser, - getByRefreshTokenAndPopulateUser, - removeByUserId +module.exports = function (sequelize, DataTypes) { + const OAuthToken = sequelize.define('OAuthToken', + { + accessToken: { + type: DataTypes.STRING + }, + accessTokenExpiresAt: { + type: DataTypes.DATE + }, + refreshToken: { + type: DataTypes.STRING + }, + refreshTokenExpiresAt: { + type: DataTypes.DATE + } + }, + { + classMethods: { + associate, + + getByRefreshTokenAndPopulateClient, + getByTokenAndPopulateUser, + getByRefreshTokenAndPopulateUser, + removeByUserId + } + } + ) + + return OAuthToken } -mongoose.model('OAuthToken', OAuthTokenSchema) +// TODO: validation +// OAuthTokenSchema.path('accessToken').required(true) +// OAuthTokenSchema.path('client').required(true) +// OAuthTokenSchema.path('user').required(true) // --------------------------------------------------------------------------- +function associate (models) { + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function getByRefreshTokenAndPopulateClient (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('client').exec().then(function (token) { + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.associations.OAuthClient ] + } + + return this.findOne(query).then(function (token) { if (!token) return token const tokenInfos = { refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, client: { - id: token.client._id.toString() + id: token.client.id }, user: { id: token.user @@ -50,13 +79,41 @@ function getByRefreshTokenAndPopulateClient (refreshToken) { } function getByTokenAndPopulateUser (bearerToken) { - return this.findOne({ accessToken: bearerToken }).populate('user').exec() + const query = { + where: { + accessToken: bearerToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + if (token) token.user = token.User + + return token + }) } function getByRefreshTokenAndPopulateUser (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('user').exec() + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + token.user = token.User + + return token + }) } function removeByUserId (userId, callback) { - return this.remove({ user: userId }, callback) + const query = { + where: { + userId: userId + } + } + + return this.destroy(query).asCallback(callback) } diff --git a/server/models/pods.js b/server/models/pods.js index 49c73472a..2c1f56203 100644 --- a/server/models/pods.js +++ b/server/models/pods.js @@ -1,79 +1,62 @@ 'use strict' -const each = require('async/each') -const mongoose = require('mongoose') const map = require('lodash/map') -const validator = require('express-validator').validator const constants = require('../initializers/constants') -const Video = mongoose.model('Video') - // --------------------------------------------------------------------------- -const PodSchema = mongoose.Schema({ - host: String, - publicKey: String, - score: { type: Number, max: constants.FRIEND_SCORE.MAX }, - createdDate: { - type: Date, - default: Date.now - } -}) - -PodSchema.path('host').validate(validator.isURL) -PodSchema.path('publicKey').required(true) -PodSchema.path('score').validate(function (value) { return !isNaN(value) }) - -PodSchema.methods = { - toFormatedJSON +module.exports = function (sequelize, DataTypes) { + const Pod = sequelize.define('Pod', + { + host: { + type: DataTypes.STRING + }, + publicKey: { + type: DataTypes.STRING(5000) + }, + score: { + type: DataTypes.INTEGER, + defaultValue: constants.FRIEND_SCORE.BASE + } + // Check createdAt + }, + { + classMethods: { + associate, + + countAll, + incrementScores, + list, + listAllIds, + listBadPods, + load, + loadByHost, + removeAll + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return Pod } -PodSchema.statics = { - countAll, - incrementScores, - list, - listAllIds, - listBadPods, - load, - loadByHost, - removeAll -} - -PodSchema.pre('save', function (next) { - const self = this - - Pod.loadByHost(this.host, function (err, pod) { - if (err) return next(err) - - if (pod) return next(new Error('Pod already exists.')) - - self.score = constants.FRIEND_SCORE.BASE - return next() - }) -}) - -PodSchema.pre('remove', function (next) { - // Remove the videos owned by this pod too - Video.listByHost(this.host, function (err, videos) { - if (err) return next(err) - - each(videos, function (video, callbackEach) { - video.remove(callbackEach) - }, next) - }) -}) - -const Pod = mongoose.model('Pod', PodSchema) +// TODO: max score -> constants.FRIENDS_SCORE.MAX +// TODO: validation +// PodSchema.path('host').validate(validator.isURL) +// PodSchema.path('publicKey').required(true) +// PodSchema.path('score').validate(function (value) { return !isNaN(value) }) // ------------------------------ METHODS ------------------------------ function toFormatedJSON () { const json = { - id: this._id, + id: this.id, host: this.host, score: this.score, - createdDate: this.createdDate + createdAt: this.createdAt } return json @@ -81,39 +64,76 @@ function toFormatedJSON () { // ------------------------------ Statics ------------------------------ +function associate (models) { + this.belongsToMany(models.Request, { + foreignKey: 'podId', + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + function countAll (callback) { - return this.count(callback) + return this.count().asCallback(callback) } function incrementScores (ids, value, callback) { if (!callback) callback = function () {} - return this.update({ _id: { $in: ids } }, { $inc: { score: value } }, { multi: true }, callback) + + const update = { + score: this.sequelize.literal('score +' + value) + } + + const query = { + where: { + id: { + $in: ids + } + } + } + + return this.update(update, query).asCallback(callback) } function list (callback) { - return this.find(callback) + return this.findAll().asCallback(callback) } function listAllIds (callback) { - return this.find({}, { _id: 1 }, function (err, pods) { + const query = { + attributes: [ 'id' ] + } + + return this.findAll(query).asCallback(function (err, pods) { if (err) return callback(err) - return callback(null, map(pods, '_id')) + return callback(null, map(pods, 'id')) }) } function listBadPods (callback) { - return this.find({ score: 0 }, callback) + const query = { + where: { + score: { $lte: 0 } + } + } + + return this.findAll(query).asCallback(callback) } function load (id, callback) { - return this.findById(id, callback) + return this.findById(id).asCallback(callback) } function loadByHost (host, callback) { - return this.findOne({ host }, callback) + const query = { + where: { + host: host + } + } + + return this.findOne(query).asCallback(callback) } function removeAll (callback) { - return this.remove({}, callback) + return this.destroy().asCallback(callback) } diff --git a/server/models/request.js b/server/models/request.js index c2cfe83ce..882f747b7 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -2,66 +2,58 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') -const values = require('lodash/values') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') const constants = require('../initializers/constants') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') - let timer = null let lastRequestTimestamp = 0 // --------------------------------------------------------------------------- -const RequestSchema = mongoose.Schema({ - request: mongoose.Schema.Types.Mixed, - endpoint: { - type: String, - enum: [ values(constants.REQUEST_ENDPOINTS) ] - }, - to: [ +module.exports = function (sequelize, DataTypes) { + const Request = sequelize.define('Request', + { + request: { + type: DataTypes.JSON + }, + endpoint: { + // TODO: enum? + type: DataTypes.STRING + } + }, { - type: mongoose.Schema.Types.ObjectId, - ref: 'Pod' + classMethods: { + associate, + + activate, + countTotalRequests, + deactivate, + flush, + forceSend, + remainingMilliSeconds + } } - ] -}) - -RequestSchema.statics = { - activate, - deactivate, - flush, - forceSend, - list, - remainingMilliSeconds -} - -RequestSchema.pre('save', function (next) { - const self = this - - if (self.to.length === 0) { - Pod.listAllIds(function (err, podIds) { - if (err) return next(err) - - // No friends - if (podIds.length === 0) return - - self.to = podIds - return next() - }) - } else { - return next() - } -}) + ) -mongoose.model('Request', RequestSchema) + return Request +} // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.belongsToMany(models.Pod, { + foreignKey: { + name: 'requestId', + allowNull: false + }, + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + function activate () { logger.info('Requests scheduler activated.') lastRequestTimestamp = Date.now() @@ -73,6 +65,14 @@ function activate () { }, constants.REQUESTS_INTERVAL) } +function countTotalRequests (callback) { + const query = { + include: [ this.sequelize.models.Pod ] + } + + return this.count(query).asCallback(callback) +} + function deactivate () { logger.info('Requests scheduler deactivated.') clearInterval(timer) @@ -90,10 +90,6 @@ function forceSend () { makeRequests.call(this) } -function list (callback) { - this.find({ }, callback) -} - function remainingMilliSeconds () { if (timer === null) return -1 @@ -136,6 +132,7 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) { // Make all the requests of the scheduler function makeRequests () { const self = this + const RequestToPod = this.sequelize.models.RequestToPod // We limit the size of the requests (REQUESTS_LIMIT) // We don't want to stuck with the same failing requests so we get a random list @@ -156,20 +153,20 @@ function makeRequests () { // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} - requests.forEach(function (poolRequest) { - poolRequest.to.forEach(function (toPodId) { - const hashKey = toPodId + poolRequest.endpoint + requests.forEach(function (request) { + request.Pods.forEach(function (toPod) { + const hashKey = toPod.id + request.endpoint if (!requestsToMakeGrouped[hashKey]) { requestsToMakeGrouped[hashKey] = { - toPodId, - endpoint: poolRequest.endpoint, - ids: [], // pool request ids, to delete them from the DB in the future + toPodId: toPod.id, + endpoint: request.endpoint, + ids: [], // request ids, to delete them from the DB in the future datas: [] // requests data, } } - requestsToMakeGrouped[hashKey].ids.push(poolRequest._id) - requestsToMakeGrouped[hashKey].datas.push(poolRequest.request) + requestsToMakeGrouped[hashKey].ids.push(request.id) + requestsToMakeGrouped[hashKey].datas.push(request.request) }) }) @@ -179,8 +176,8 @@ function makeRequests () { eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, function (hashKey, callbackEach) { const requestToMake = requestsToMakeGrouped[hashKey] - // FIXME: mongodb request inside a loop :/ - Pod.load(requestToMake.toPodId, function (err, toPod) { + // FIXME: SQL request inside a loop :/ + self.sequelize.models.Pod.load(requestToMake.toPodId, function (err, toPod) { if (err) { logger.error('Error finding pod by id.', { err: err }) return callbackEach() @@ -191,7 +188,7 @@ function makeRequests () { const requestIdsToDelete = requestToMake.ids logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPodId) - removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) + RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) return callbackEach() } @@ -202,7 +199,7 @@ function makeRequests () { goodPods.push(requestToMake.toPodId) // Remove the pod id of these request ids - removePodOf.call(self, requestToMake.ids, requestToMake.toPodId, callbackEach) + RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPodId, callbackEach) } else { badPods.push(requestToMake.toPodId) callbackEach() @@ -211,18 +208,22 @@ function makeRequests () { }) }, function () { // All the requests were made, we update the pods score - updatePodsScore(goodPods, badPods) + updatePodsScore.call(self, goodPods, badPods) // Flush requests with no pod - removeWithEmptyTo.call(self) + removeWithEmptyTo.call(self, function (err) { + if (err) logger.error('Error when removing requests with no pods.', { error: err }) + }) }) }) } // Remove pods with a score of 0 (too many requests where they were unreachable) function removeBadPods () { + const self = this + waterfall([ function findBadPods (callback) { - Pod.listBadPods(function (err, pods) { + self.sequelize.models.Pod.listBadPods(function (err, pods) { if (err) { logger.error('Cannot find bad pods.', { error: err }) return callback(err) @@ -233,10 +234,8 @@ function removeBadPods () { }, function removeTheseBadPods (pods, callback) { - if (pods.length === 0) return callback(null, 0) - each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, function (err) { return callback(err, pods.length) }) @@ -253,43 +252,67 @@ function removeBadPods () { } function updatePodsScore (goodPods, badPods) { + const self = this + const Pod = this.sequelize.models.Pod + logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) - Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.') - }) + if (goodPods.length !== 0) { + Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { + if (err) logger.error('Cannot increment scores of good pods.') + }) + } - Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.') - removeBadPods() - }) + if (badPods.length !== 0) { + Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { + if (err) logger.error('Cannot decrement scores of bad pods.') + removeBadPods.call(self) + }) + } } function listWithLimitAndRandom (limit, callback) { const self = this - self.count(function (err, count) { + self.count().asCallback(function (err, count) { if (err) return callback(err) + // Optimization... + if (count === 0) return callback(null, []) + let start = Math.floor(Math.random() * count) - limit if (start < 0) start = 0 - self.find().sort({ _id: 1 }).skip(start).limit(limit).exec(callback) + const query = { + order: [ + [ 'id', 'ASC' ] + ], + offset: start, + limit: limit, + include: [ this.sequelize.models.Pod ] + } + + self.findAll(query).asCallback(callback) }) } function removeAll (callback) { - this.remove({ }, callback) -} - -function removePodOf (requestsIds, podId, callback) { - if (!callback) callback = function () {} - - this.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback) + // Delete all requests + this.destroy({ truncate: true }).asCallback(callback) } function removeWithEmptyTo (callback) { if (!callback) callback = function () {} - this.remove({ to: { $size: 0 } }, callback) + const query = { + where: { + id: { + $notIn: [ + this.sequelize.literal('SELECT "requestId" FROM "RequestToPods"') + ] + } + } + } + + this.destroy(query).asCallback(callback) } diff --git a/server/models/requestToPod.js b/server/models/requestToPod.js new file mode 100644 index 000000000..378c2bdcf --- /dev/null +++ b/server/models/requestToPod.js @@ -0,0 +1,30 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestToPod = sequelize.define('RequestToPod', {}, { + classMethods: { + removePodOf + } + }) + + return RequestToPod +} + +// --------------------------------------------------------------------------- + +function removePodOf (requestsIds, podId, callback) { + if (!callback) callback = function () {} + + const query = { + where: { + requestId: { + $in: requestsIds + }, + podId: podId + } + } + + this.destroy(query).asCallback(callback) +} diff --git a/server/models/user.js b/server/models/user.js index a19de7072..e50eb96ea 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,60 +1,60 @@ -const mongoose = require('mongoose') - -const customUsersValidators = require('../helpers/custom-validators').users const modelUtils = require('./utils') const peertubeCrypto = require('../helpers/peertube-crypto') -const OAuthToken = mongoose.model('OAuthToken') - // --------------------------------------------------------------------------- -const UserSchema = mongoose.Schema({ - createdDate: { - type: Date, - default: Date.now - }, - password: String, - username: String, - role: String -}) - -UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) -UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) -UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) - -UserSchema.methods = { - isPasswordMatch, - toFormatedJSON +module.exports = function (sequelize, DataTypes) { + const User = sequelize.define('User', + { + password: { + type: DataTypes.STRING + }, + username: { + type: DataTypes.STRING + }, + role: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate, + + countTotal, + getByUsername, + list, + listForApi, + loadById, + loadByUsername + }, + instanceMethods: { + isPasswordMatch, + toFormatedJSON + }, + hooks: { + beforeCreate: beforeCreateOrUpdate, + beforeUpdate: beforeCreateOrUpdate + } + } + ) + + return User } -UserSchema.statics = { - countTotal, - getByUsername, - list, - listForApi, - loadById, - loadByUsername -} +// TODO: Validation +// UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) +// UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) +// UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) -UserSchema.pre('save', function (next) { - const user = this - - peertubeCrypto.cryptPassword(this.password, function (err, hash) { +function beforeCreateOrUpdate (user, options, next) { + peertubeCrypto.cryptPassword(user.password, function (err, hash) { if (err) return next(err) user.password = hash return next() }) -}) - -UserSchema.pre('remove', function (next) { - const user = this - - OAuthToken.removeByUserId(user._id, next) -}) - -mongoose.model('User', UserSchema) +} // ------------------------------ METHODS ------------------------------ @@ -64,35 +64,63 @@ function isPasswordMatch (password, callback) { function toFormatedJSON () { return { - id: this._id, + id: this.id, username: this.username, role: this.role, - createdDate: this.createdDate + createdAt: this.createdAt } } // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.hasMany(models.OAuthToken, { + foreignKey: 'userId', + onDelete: 'cascade' + }) +} + function countTotal (callback) { - return this.count(callback) + return this.count().asCallback(callback) } function getByUsername (username) { - return this.findOne({ username: username }) + const query = { + where: { + username: username + } + } + + return this.findOne(query) } function list (callback) { - return this.find(callback) + return this.find().asCallback(callback) } function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } function loadById (id, callback) { - return this.findById(id, callback) + return this.findById(id).asCallback(callback) } function loadByUsername (username, callback) { - return this.findOne({ username: username }, callback) + const query = { + where: { + username: username + } + } + + return this.findOne(query).asCallback(callback) } diff --git a/server/models/utils.js b/server/models/utils.js index e798aabe6..49636b3d8 100644 --- a/server/models/utils.js +++ b/server/models/utils.js @@ -1,28 +1,23 @@ 'use strict' -const parallel = require('async/parallel') - const utils = { - listForApiWithCount + getSort } -function listForApiWithCount (query, start, count, sort, callback) { - const self = this +// Translate for example "-name" to [ 'name', 'DESC' ] +function getSort (value) { + let field + let direction - parallel([ - function (asyncCallback) { - self.find(query).skip(start).limit(count).sort(sort).exec(asyncCallback) - }, - function (asyncCallback) { - self.count(query, asyncCallback) - } - ], function (err, results) { - if (err) return callback(err) + if (value.substring(0, 1) === '-') { + direction = 'DESC' + field = value.substring(1) + } else { + direction = 'ASC' + field = value + } - const data = results[0] - const total = results[1] - return callback(null, data, total) - }) + return [ field, direction ] } // --------------------------------------------------------------------------- diff --git a/server/models/video.js b/server/models/video.js index 330067cdf..8ef07c9e6 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -7,102 +7,93 @@ const magnetUtil = require('magnet-uri') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') -const mongoose = require('mongoose') const constants = require('../initializers/constants') -const customVideosValidators = require('../helpers/custom-validators').videos const logger = require('../helpers/logger') const modelUtils = require('./utils') // --------------------------------------------------------------------------- +module.exports = function (sequelize, DataTypes) { // TODO: add indexes on searchable columns -const VideoSchema = mongoose.Schema({ - name: String, - extname: { - type: String, - enum: [ '.mp4', '.webm', '.ogv' ] - }, - remoteId: mongoose.Schema.Types.ObjectId, - description: String, - magnet: { - infoHash: String - }, - podHost: String, - author: String, - duration: Number, - tags: [ String ], - createdDate: { - type: Date, - default: Date.now - } -}) - -VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) -VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) -VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) -VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) -VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) -VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) - -VideoSchema.methods = { - generateMagnetUri, - getVideoFilename, - getThumbnailName, - getPreviewName, - getTorrentName, - isOwned, - toFormatedJSON, - toRemoteJSON -} - -VideoSchema.statics = { - generateThumbnailFromBase64, - getDurationFromFile, - listForApi, - listByHostAndRemoteId, - listByHost, - listOwned, - listOwnedByAuthor, - listRemotes, - load, - search -} - -VideoSchema.pre('remove', function (next) { - const video = this - const tasks = [] - - tasks.push( - function (callback) { - removeThumbnail(video, callback) - } - ) - - if (video.isOwned()) { - tasks.push( - function (callback) { - removeFile(video, callback) + const Video = sequelize.define('Video', + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true }, - function (callback) { - removeTorrent(video, callback) + name: { + type: DataTypes.STRING }, - function (callback) { - removePreview(video, callback) + extname: { + // TODO: enum? + type: DataTypes.STRING + }, + remoteId: { + type: DataTypes.UUID + }, + description: { + type: DataTypes.STRING + }, + infoHash: { + type: DataTypes.STRING + }, + duration: { + type: DataTypes.INTEGER + }, + tags: { + type: DataTypes.ARRAY(DataTypes.STRING) } - ) - } + }, + { + classMethods: { + associate, + + generateThumbnailFromBase64, + getDurationFromFile, + listForApi, + listByHostAndRemoteId, + listOwnedAndPopulateAuthor, + listOwnedByAuthor, + load, + loadAndPopulateAuthor, + loadAndPopulateAuthorAndPod, + searchAndPopulateAuthorAndPod + }, + instanceMethods: { + generateMagnetUri, + getVideoFilename, + getThumbnailName, + getPreviewName, + getTorrentName, + isOwned, + toFormatedJSON, + toRemoteJSON + }, + hooks: { + beforeCreate, + afterDestroy + } + } + ) - parallel(tasks, next) -}) + return Video +} -VideoSchema.pre('save', function (next) { - const video = this +// TODO: Validation +// VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) +// VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) +// VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) +// VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) +// VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) +// VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) + +function beforeCreate (video, options, next) { const tasks = [] if (video.isOwned()) { const videoPath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) - this.podHost = constants.CONFIG.WEBSERVER.HOST tasks.push( // TODO: refractoring @@ -123,7 +114,7 @@ VideoSchema.pre('save', function (next) { if (err) return callback(err) const parsedTorrent = parseTorrent(torrent) - video.magnet.infoHash = parsedTorrent.infoHash + video.infoHash = parsedTorrent.infoHash callback(null) }) @@ -141,12 +132,46 @@ VideoSchema.pre('save', function (next) { } return next() -}) +} + +function afterDestroy (video, options, next) { + const tasks = [] -mongoose.model('Video', VideoSchema) + tasks.push( + function (callback) { + removeThumbnail(video, callback) + } + ) + + if (video.isOwned()) { + tasks.push( + function (callback) { + removeFile(video, callback) + }, + function (callback) { + removeTorrent(video, callback) + }, + function (callback) { + removePreview(video, callback) + } + ) + } + + parallel(tasks, next) +} // ------------------------------ METHODS ------------------------------ +function associate (models) { + this.belongsTo(models.Author, { + foreignKey: { + name: 'authorId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function generateMagnetUri () { let baseUrlHttp, baseUrlWs @@ -154,8 +179,8 @@ function generateMagnetUri () { baseUrlHttp = constants.CONFIG.WEBSERVER.URL baseUrlWs = constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT } else { - baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.podHost - baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.podHost + baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host + baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.Author.Pod.host } const xs = baseUrlHttp + constants.STATIC_PATHS.TORRENTS + this.getTorrentName() @@ -166,7 +191,7 @@ function generateMagnetUri () { xs, announce, urlList, - infoHash: this.magnet.infoHash, + infoHash: this.infoHash, name: this.name } @@ -174,20 +199,20 @@ function generateMagnetUri () { } function getVideoFilename () { - if (this.isOwned()) return this._id + this.extname + if (this.isOwned()) return this.id + this.extname return this.remoteId + this.extname } function getThumbnailName () { // We always have a copy of the thumbnail - return this._id + '.jpg' + return this.id + '.jpg' } function getPreviewName () { const extension = '.jpg' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -195,7 +220,7 @@ function getPreviewName () { function getTorrentName () { const extension = '.torrent' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -205,18 +230,27 @@ function isOwned () { } function toFormatedJSON () { + let podHost + + if (this.Author.Pod) { + podHost = this.Author.Pod.host + } else { + // It means it's our video + podHost = constants.CONFIG.WEBSERVER.HOST + } + const json = { - id: this._id, + id: this.id, name: this.name, description: this.description, - podHost: this.podHost, + podHost, isLocal: this.isOwned(), magnetUri: this.generateMagnetUri(), - author: this.author, + author: this.Author.name, duration: this.duration, tags: this.tags, thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), - createdDate: this.createdDate + createdAt: this.createdAt } return json @@ -236,13 +270,13 @@ function toRemoteJSON (callback) { const remoteVideo = { name: self.name, description: self.description, - magnet: self.magnet, - remoteId: self._id, - author: self.author, + infoHash: self.infoHash, + remoteId: self.id, + author: self.Author.name, duration: self.duration, thumbnailBase64: new Buffer(thumbnailData).toString('base64'), tags: self.tags, - createdDate: self.createdDate, + createdAt: self.createdAt, extname: self.extname } @@ -273,50 +307,168 @@ function getDurationFromFile (videoPath, callback) { } function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ], + include: [ + { + model: this.sequelize.models.Author, + include: [ this.sequelize.models.Pod ] + } + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } function listByHostAndRemoteId (fromHost, remoteId, callback) { - this.find({ podHost: fromHost, remoteId: remoteId }, callback) -} + const query = { + where: { + remoteId: remoteId + }, + include: [ + { + model: this.sequelize.models.Author, + include: [ + { + model: this.sequelize.models.Pod, + where: { + host: fromHost + } + } + ] + } + ] + } -function listByHost (fromHost, callback) { - this.find({ podHost: fromHost }, callback) + return this.findAll(query).asCallback(callback) } -function listOwned (callback) { +function listOwnedAndPopulateAuthor (callback) { // If remoteId is null this is *our* video - this.find({ remoteId: null }, callback) + const query = { + where: { + remoteId: null + }, + include: [ this.sequelize.models.Author ] + } + + return this.findAll(query).asCallback(callback) } function listOwnedByAuthor (author, callback) { - this.find({ remoteId: null, author: author }, callback) -} + const query = { + where: { + remoteId: null + }, + include: [ + { + model: this.sequelize.models.Author, + where: { + name: author + } + } + ] + } -function listRemotes (callback) { - this.find({ remoteId: { $ne: null } }, callback) + return this.findAll(query).asCallback(callback) } function load (id, callback) { - this.findById(id, callback) + return this.findById(id).asCallback(callback) } -function search (value, field, start, count, sort, callback) { - const query = {} +function loadAndPopulateAuthor (id, callback) { + const options = { + include: [ this.sequelize.models.Author ] + } + + return this.findById(id, options).asCallback(callback) +} + +function loadAndPopulateAuthorAndPod (id, callback) { + const options = { + include: [ + { + model: this.sequelize.models.Author, + include: [ this.sequelize.models.Pod ] + } + ] + } + + return this.findById(id, options).asCallback(callback) +} + +function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callback) { + const podInclude = { + model: this.sequelize.models.Pod + } + const authorInclude = { + model: this.sequelize.models.Author, + include: [ + podInclude + ] + } + + const query = { + where: {}, + include: [ + authorInclude + ], + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ] + } + + // TODO: include our pod for podHost searches (we are not stored in the database) // Make an exact search with the magnet if (field === 'magnetUri') { const infoHash = magnetUtil.decode(value).infoHash - query.magnet = { - infoHash - } + query.where.infoHash = infoHash } else if (field === 'tags') { - query[field] = value + query.where[field] = value + } else if (field === 'host') { + const whereQuery = { + '$Author.Pod.host$': { + $like: '%' + value + '%' + } + } + + // Include our pod? (not stored in the database) + if (constants.CONFIG.WEBSERVER.HOST.indexOf(value) !== -1) { + query.where = { + $or: [ + whereQuery, + { + remoteId: null + } + ] + } + } else { + query.where = whereQuery + } + } else if (field === 'author') { + query.where = { + '$Author.name$': { + $like: '%' + value + '%' + } + } } else { - query[field] = new RegExp(value, 'i') + query.where[field] = { + $like: '%' + value + '%' + } } - modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } // --------------------------------------------------------------------------- diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js index 444c2fc55..d9e51770c 100644 --- a/server/tests/api/check-params.js +++ b/server/tests/api/check-params.js @@ -465,7 +465,7 @@ describe('Test parameters validator', function () { it('Should return 404 with an incorrect video', function (done) { request(server.url) - .get(path + '123456789012345678901234') + .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') .set('Accept', 'application/json') .expect(404, done) }) @@ -490,7 +490,7 @@ describe('Test parameters validator', function () { it('Should fail with a video which does not exist', function (done) { request(server.url) - .delete(path + '123456789012345678901234') + .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') .set('Authorization', 'Bearer ' + server.accessToken) .expect(404, done) }) @@ -711,7 +711,7 @@ describe('Test parameters validator', function () { it('Should return 404 with a non existing id', function (done) { request(server.url) - .delete(path + '579f982228c99c221d8092b8') + .delete(path + '45') .set('Authorization', 'Bearer ' + server.accessToken) .expect(404, done) }) diff --git a/server/tests/api/friends-basic.js b/server/tests/api/friends-basic.js index a871f9838..3a904dbd7 100644 --- a/server/tests/api/friends-basic.js +++ b/server/tests/api/friends-basic.js @@ -97,7 +97,7 @@ describe('Test basic friends', function () { const pod = result[0] expect(pod.host).to.equal(servers[2].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) @@ -114,7 +114,7 @@ describe('Test basic friends', function () { const pod = result[0] expect(pod.host).to.equal(servers[1].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index be278d7c5..f0fe59c5f 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -104,7 +104,7 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(10) expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9001') { @@ -166,7 +166,7 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(5) expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9002') { @@ -246,7 +246,7 @@ describe('Test multiple pods', function () { expect(video1.duration).to.equal(5) expect(video1.tags).to.deep.equal([ 'tag1p3' ]) expect(video1.author).to.equal('root') - expect(miscsUtils.dateIsValid(video1.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video1.createdAt)).to.be.true expect(video2.name).to.equal('my super name for pod 3-2') expect(video2.description).to.equal('my super description for pod 3-2') @@ -255,7 +255,7 @@ describe('Test multiple pods', function () { expect(video2.duration).to.equal(5) expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ]) expect(video2.author).to.equal('root') - expect(miscsUtils.dateIsValid(video2.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video2.createdAt)).to.be.true if (server.url !== 'http://localhost:9003') { expect(video1.isLocal).to.be.false diff --git a/server/tests/api/requests.js b/server/tests/api/requests.js index af36f6e34..7e790b54b 100644 --- a/server/tests/api/requests.js +++ b/server/tests/api/requests.js @@ -69,7 +69,7 @@ describe('Test requests stats', function () { }) }) - it('Should have the correct request', function (done) { + it('Should have the correct total request', function (done) { this.timeout(15000) const server = servers[0] @@ -83,11 +83,7 @@ describe('Test requests stats', function () { if (err) throw err const body = res.body - expect(body.requests).to.have.lengthOf(1) - - const request = body.requests[0] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') + expect(body.totalRequests).to.equal(1) // Wait one cycle setTimeout(done, 10000) @@ -95,27 +91,6 @@ describe('Test requests stats', function () { }) }) - it('Should have the correct requests', function (done) { - const server = servers[0] - - uploadVideo(server, function (err) { - if (err) throw err - - getRequestsStats(server, function (err, res) { - if (err) throw err - - const body = res.body - expect(body.requests).to.have.lengthOf(2) - - const request = body.requests[1] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') - - done() - }) - }) - }) - after(function (done) { process.kill(-servers[0].app.pid) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 65d1a7a65..aedecacf3 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -82,7 +82,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -116,7 +116,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -142,7 +142,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -154,7 +154,7 @@ describe('Test a single pod', function () { }) it('Should search the video by podHost', function (done) { - videosUtils.searchVideo(server.url, '9001', 'podHost', function (err, res) { + videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { if (err) throw err expect(res.body.total).to.equal(1) @@ -168,7 +168,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -194,7 +194,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -425,7 +425,7 @@ describe('Test a single pod', function () { }) it('Should search all the 9001 port videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, '9001', 'podHost', 0, 15, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { if (err) throw err const videos = res.body.data @@ -437,7 +437,7 @@ describe('Test a single pod', function () { }) it('Should search all the localhost videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'localhost', 'podHost', 0, 15, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { if (err) throw err const videos = res.body.data diff --git a/server/tests/api/users.js b/server/tests/api/users.js index 94267f104..e6d937eb0 100644 --- a/server/tests/api/users.js +++ b/server/tests/api/users.js @@ -261,8 +261,8 @@ describe('Test users', function () { }) }) - it('Should list only the second user by createdDate desc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdDate', function (err, res) { + it('Should list only the second user by createdAt desc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdAt', function (err, res) { if (err) throw err const result = res.body @@ -279,8 +279,8 @@ describe('Test users', function () { }) }) - it('Should list all the users by createdDate asc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdDate', function (err, res) { + it('Should list all the users by createdAt asc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdAt', function (err, res) { if (err) throw err const result = res.body diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index 01c9a2f39..4e55f8f5c 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -60,12 +60,12 @@ function runServer (number, callback) { // These actions are async so we need to be sure that they have both been done const serverRunString = { - 'Connected to mongodb': false, + 'Database is ready': false, 'Server listening on port': false } const regexps = { - client_id: 'Client id: ([a-f0-9]+)', + client_id: 'Client id: (.+)', client_secret: 'Client secret: (.+)', user_username: 'Username: (.+)', user_password: 'User password: (.+)' diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 536093db1..5c120597f 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -25,7 +25,7 @@ function getAllVideosListBy (url, end) { request(url) .get(path) - .query({ sort: 'createdDate' }) + .query({ sort: 'createdAt' }) .query({ start: 0 }) .query({ count: 10000 }) .set('Accept', 'application/json') -- 2.25.1