From 23687332e6d7972c6a0eee30ce9056e45dec022b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 21 Jun 2018 18:29:28 +0200 Subject: [PATCH] Improve update host script and add warning if AP urls are invalid --- scripts/update-host.ts | 141 ++++++++++++++++++++++++++------ server.ts | 11 ++- server/initializers/checker.ts | 25 +++++- server/tests/cli/update-host.ts | 65 ++++++++++++++- support/doc/tools.md | 9 ++ support/doc/translation.md | 5 +- 6 files changed, 222 insertions(+), 34 deletions(-) diff --git a/scripts/update-host.ts b/scripts/update-host.ts index ed8b999a9..1dc19664d 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts @@ -1,35 +1,126 @@ -import { getServerActor } from '../server/helpers/utils' -import { initDatabaseModels } from '../server/initializers' +import { CONFIG, initDatabaseModels } from '../server/initializers' import { ActorFollowModel } from '../server/models/activitypub/actor-follow' import { VideoModel } from '../server/models/video/video' +import { ActorModel } from '../server/models/activitypub/actor' +import { + getAccountActivityPubUrl, + getAnnounceActivityPubUrl, + getVideoActivityPubUrl, getVideoChannelActivityPubUrl, + getVideoCommentActivityPubUrl +} from '../server/lib/activitypub' +import { VideoShareModel } from '../server/models/video/video-share' +import { VideoCommentModel } from '../server/models/video/video-comment' +import { getServerActor } from '../server/helpers/utils' +import { AccountModel } from '../server/models/account/account' +import { VideoChannelModel } from '../server/models/video/video-channel' -initDatabaseModels(true) - .then(() => { - return getServerActor() - }) - .then(serverAccount => { - return ActorFollowModel.listAcceptedFollowingUrlsForApi([ serverAccount.id ], undefined) - }) - .then(res => { - return res.total > 0 +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) }) - .then(hasFollowing => { + +async function run () { + await initDatabaseModels(true) + + const serverAccount = await getServerActor() + + { + const res = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ serverAccount.id ], undefined) + const hasFollowing = res.total > 0 + if (hasFollowing === true) { - console.log('Cannot update host because you follow other servers!') - process.exit(-1) + throw new Error('Cannot update host because you follow other servers!') } + } - console.log('Updating torrent files.') - return VideoModel.list() - }) - .then(async videos => { - for (const video of videos) { - for (const file of video.VideoFiles) { - await video.createTorrentAndSetInfoHash(file) - console.log('Updated video ' + video.uuid) + console.log('Updating actors.') + + const actors: ActorModel[] = await ActorModel.unscoped().findAll({ + include: [ + { + model: VideoChannelModel.unscoped(), + required: false + }, + { + model: AccountModel.unscoped(), + required: false } - } + ] + }) + for (const actor of actors) { + if (actor.isOwned() === false) continue + + console.log('Updating actor ' + actor.url) + + const newUrl = actor.Account + ? getAccountActivityPubUrl(actor.preferredUsername) + : getVideoChannelActivityPubUrl(actor.preferredUsername) + + actor.url = newUrl + actor.inboxUrl = newUrl + '/inbox' + actor.outboxUrl = newUrl + '/outbox' + actor.sharedInboxUrl = CONFIG.WEBSERVER.URL + '/inbox' + actor.followersUrl = newUrl + '/followers' + actor.followingUrl = newUrl + '/following' + + await actor.save() + } + + console.log('Updating video shares.') + + const videoShares: VideoShareModel[] = await VideoShareModel.findAll({ + include: [ VideoModel.unscoped(), ActorModel.unscoped() ] }) - .then(() => { - process.exit(0) + for (const videoShare of videoShares) { + if (videoShare.Video.isOwned() === false) continue + + console.log('Updating video share ' + videoShare.url) + + videoShare.url = getAnnounceActivityPubUrl(videoShare.Video.url, videoShare.Actor) + await videoShare.save() + } + + console.log('Updating video comments.') + const videoComments: VideoCommentModel[] = await VideoCommentModel.findAll({ + include: [ + { + model: VideoModel.unscoped() + }, + { + model: AccountModel.unscoped(), + include: [ + { + model: ActorModel.unscoped() + } + ] + } + ] }) + for (const comment of videoComments) { + if (comment.isOwned() === false) continue + + console.log('Updating comment ' + comment.url) + + comment.url = getVideoCommentActivityPubUrl(comment.Video, comment) + await comment.save() + } + + console.log('Updating video and torrent files.') + + const videos = await VideoModel.list() + for (const video of videos) { + if (video.isOwned() === false) continue + + console.log('Updated video ' + video.uuid) + + video.url = getVideoActivityPubUrl(video) + await video.save() + + for (const file of video.VideoFiles) { + console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) + await video.createTorrentAndSetInfoHash(file) + } + } +} diff --git a/server.ts b/server.ts index ef89ff5f6..f756bf9d4 100644 --- a/server.ts +++ b/server.ts @@ -1,6 +1,4 @@ // FIXME: https://github.com/nodejs/node/pull/16853 -import { ScheduleVideoUpdateModel } from './server/models/video/schedule-video-update' - require('tls').DEFAULT_ECDH_CURVE = 'auto' import { isTestInstance } from './server/helpers/core-utils' @@ -26,7 +24,7 @@ process.title = 'peertube' const app = express() // ----------- Core checker ----------- -import { checkMissedConfig, checkFFmpeg, checkConfig } from './server/initializers/checker' +import { checkMissedConfig, checkFFmpeg, checkConfig, checkActivityPubUrls } from './server/initializers/checker' // Do not use barrels because we don't want to load all modules here (we need to initialize database first) import { logger } from './server/helpers/logger' @@ -191,6 +189,13 @@ async function startApplication () { await installApplication() + // Check activity pub urls are valid + checkActivityPubUrls() + .catch(err => { + logger.error('Error in ActivityPub URLs checker.', { err }) + process.exit(-1) + }) + // Email initialization Emailer.Instance.init() await Emailer.Instance.checkConnectionOrDie() diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 6259c7b6c..d5402098f 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -3,6 +3,28 @@ import { promisify0 } from '../helpers/core-utils' import { UserModel } from '../models/account/user' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' +import { parse } from 'url' +import { CONFIG } from './constants' +import { logger } from '../helpers/logger' +import { getServerActor } from '../helpers/utils' + +async function checkActivityPubUrls () { + const actor = await getServerActor() + + const parsed = parse(actor.url) + if (CONFIG.WEBSERVER.HOST !== parsed.host) { + const NODE_ENV = config.util.getEnv('NODE_ENV') + const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR') + + logger.warn( + 'It seems PeerTube was started (and created some data) with another domain name. ' + + 'This means you will not be able to federate! ' + + 'Please use %s %s npm run update-host to fix this.', + NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '', + NODE_ENV ? `NODE_ENV=${NODE_ENV}` : '' + ) + } +} // Some checks on configuration files // Return an error message, or null if everything is okay @@ -95,5 +117,6 @@ export { checkMissedConfig, clientsExist, usersExist, - applicationExist + applicationExist, + checkActivityPubUrls } diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts index d0c6d2042..968b7bd7f 100644 --- a/server/tests/cli/update-host.ts +++ b/server/tests/cli/update-host.ts @@ -3,20 +3,26 @@ import 'mocha' import * as chai from 'chai' import { VideoDetails } from '../../../shared/models/videos' +import { waitJobs } from '../utils/server/jobs' +import { addVideoCommentThread } from '../utils/videos/video-comments' import { + addVideoChannel, + createUser, execCLI, flushTests, getEnvCli, getVideo, + getVideoChannelsList, getVideosList, killallServers, + makeActivityPubGetRequest, parseTorrentVideo, runServer, ServerInfo, setAccessTokensToServers, uploadVideo } from '../utils' -import { waitJobs } from '../utils/server/jobs' +import { getAccountsList } from '../utils/users/accounts' const expect = chai.expect @@ -39,13 +45,28 @@ describe('Test update host scripts', function () { // Upload two videos for our needs const videoAttributes = {} + const resVideo1 = await uploadVideo(server.url, server.accessToken, videoAttributes) + const video1UUID = resVideo1.body.video.uuid await uploadVideo(server.url, server.accessToken, videoAttributes) - await uploadVideo(server.url, server.accessToken, videoAttributes) + + // Create a user + await createUser(server.url, server.accessToken, 'toto', 'coucou') + + // Create channel + const videoChannel = { + displayName: 'second video channel', + description: 'super video channel description' + } + await addVideoChannel(server.url, server.accessToken, videoChannel) + + // Create comments + const text = 'my super first comment' + await addVideoCommentThread(server.url, server.accessToken, video1UUID, text) await waitJobs(server) }) - it('Should update torrent hosts', async function () { + it('Should run update host', async function () { this.timeout(30000) killallServers([ server ]) @@ -54,6 +75,44 @@ describe('Test update host scripts', function () { const env = getEnvCli(server) await execCLI(`${env} npm run update-host`) + }) + + it('Should have updated videos url', async function () { + const res = await getVideosList(server.url) + expect(res.body.total).to.equal(2) + + for (const video of res.body.data) { + const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) + + expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) + } + }) + + it('Should have updated video channels url', async function () { + const res = await getVideoChannelsList(server.url, 0, 5, '-name') + expect(res.body.total).to.equal(3) + + for (const channel of res.body.data) { + const { body } = await makeActivityPubGetRequest(server.url, '/video-channels/' + channel.uuid) + + expect(body.id).to.equal('http://localhost:9002/video-channels/' + channel.uuid) + } + }) + + it('Should have update accounts url', async function () { + const res = await getAccountsList(server.url) + expect(res.body.total).to.equal(3) + + for (const account of res.body.data) { + const usernameWithDomain = account.name + const { body } = await makeActivityPubGetRequest(server.url, '/accounts/' + usernameWithDomain) + + expect(body.id).to.equal('http://localhost:9002/accounts/' + usernameWithDomain) + } + }) + + it('Should update torrent hosts', async function () { + this.timeout(30000) const res = await getVideosList(server.url) const videos = res.body.data diff --git a/support/doc/tools.md b/support/doc/tools.md index 0addc0803..cb3944595 100644 --- a/support/doc/tools.md +++ b/support/doc/tools.md @@ -113,4 +113,13 @@ To delete them (a confirmation will be demanded first): ``` $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage +``` + +### update-host.js + +If you started PeerTube with a domain, and then changed it you will have invalid torrent files and invalid URLs in your database. +To fix this, you have to run: + +``` +$ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-host ``` \ No newline at end of file diff --git a/support/doc/translation.md b/support/doc/translation.md index 3ad239278..52045083c 100644 --- a/support/doc/translation.md +++ b/support/doc/translation.md @@ -18,7 +18,8 @@ If you don't see your locale in the platform, please [create an issue](https://g There are 4 files: * **angular**: contains client strings - * **player**: contains player strings - * **server**: contains server strings (language, licence...) + * **player**: contains player strings. + Most of the strings come from VideoJS, so you can help yourself by using [video.js JSON files](https://github.com/videojs/video.js/tree/master/lang) + * **server**: contains server strings (privacies, licences...) * **iso639**: contains iso639 (languages) strings used by PeerTube to describe the audio language of a particular video. It's the reason why these strings should be translated too. There are many strings so do not hesitate to translate only main audio languages. -- 2.25.1