From 2feebf3e6afaad9ab80976d1557d3a7bcf94de03 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 5 Dec 2018 17:27:24 +0100 Subject: [PATCH] Add sitemap --- package.json | 1 + scripts/clean/server/test.sh | 1 + server.ts | 3 +- server/controllers/bots.ts | 101 +++++++++++++++++++++++++++ server/controllers/index.ts | 1 + server/helpers/express-utils.ts | 4 +- server/initializers/constants.ts | 1 + server/models/account/account.ts | 21 ++++++ server/models/video/video-channel.ts | 21 ++++++ server/tests/misc-endpoints.ts | 72 ++++++++++++++++++- yarn.lock | 19 +++++ 11 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 server/controllers/bots.ts diff --git a/package.json b/package.json index c5e4c329c..aa4f447aa 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "sequelize": "4.41.2", "sequelize-typescript": "0.6.6", "sharp": "^0.21.0", + "sitemap": "^2.1.0", "srt-to-vtt": "^1.1.2", "summon-install": "^0.4.3", "useragent": "^2.3.0", diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 235ff52cc..b897c30ba 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -18,6 +18,7 @@ removeFiles () { dropRedis () { redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL + redis-cli KEYS "redis-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL } for i in $(seq 1 6); do diff --git a/server.ts b/server.ts index 3025a6fd7..4a2a6ddf5 100644 --- a/server.ts +++ b/server.ts @@ -87,7 +87,7 @@ import { servicesRouter, webfingerRouter, trackerRouter, - createWebsocketServer + createWebsocketServer, botsRouter } from './server/controllers' import { advertiseDoNotTrack } from './server/middlewares/dnt' import { Redis } from './server/lib/redis' @@ -156,6 +156,7 @@ app.use('/', activityPubRouter) app.use('/', feedsRouter) app.use('/', webfingerRouter) app.use('/', trackerRouter) +app.use('/', botsRouter) // Static files app.use('/', staticRouter) diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts new file mode 100644 index 000000000..b4eaccf9f --- /dev/null +++ b/server/controllers/bots.ts @@ -0,0 +1,101 @@ +import * as express from 'express' +import { asyncMiddleware } from '../middlewares' +import { CONFIG, ROUTE_CACHE_LIFETIME } from '../initializers' +import * as sitemapModule from 'sitemap' +import { logger } from '../helpers/logger' +import { VideoModel } from '../models/video/video' +import { VideoChannelModel } from '../models/video/video-channel' +import { AccountModel } from '../models/account/account' +import { cacheRoute } from '../middlewares/cache' +import { buildNSFWFilter } from '../helpers/express-utils' +import { truncate } from 'lodash' + +const botsRouter = express.Router() + +// Special route that add OpenGraph and oEmbed tags +// Do not use a template engine for a so little thing +botsRouter.use('/sitemap.xml', + asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP)), + asyncMiddleware(getSitemap) +) + +// --------------------------------------------------------------------------- + +export { + botsRouter +} + +// --------------------------------------------------------------------------- + +async function getSitemap (req: express.Request, res: express.Response) { + let urls = getSitemapBasicUrls() + + urls = urls.concat(await getSitemapLocalVideoUrls()) + urls = urls.concat(await getSitemapVideoChannelUrls()) + urls = urls.concat(await getSitemapAccountUrls()) + + const sitemap = sitemapModule.createSitemap({ + hostname: CONFIG.WEBSERVER.URL, + urls: urls + }) + + sitemap.toXML((err, xml) => { + if (err) { + logger.error('Cannot generate sitemap.', { err }) + return res.sendStatus(500) + } + + res.header('Content-Type', 'application/xml') + res.send(xml) + }) +} + +async function getSitemapVideoChannelUrls () { + const rows = await VideoChannelModel.listLocalsForSitemap('createdAt') + + return rows.map(channel => ({ + url: CONFIG.WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername + })) +} + +async function getSitemapAccountUrls () { + const rows = await AccountModel.listLocalsForSitemap('createdAt') + + return rows.map(channel => ({ + url: CONFIG.WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername + })) +} + +async function getSitemapLocalVideoUrls () { + const resultList = await VideoModel.listForApi({ + start: 0, + count: undefined, + sort: 'createdAt', + includeLocalVideos: true, + nsfw: buildNSFWFilter(), + filter: 'local', + withFiles: false + }) + + return resultList.data.map(v => ({ + url: CONFIG.WEBSERVER.URL + '/videos/watch/' + v.uuid, + video: [ + { + title: v.name, + // Sitemap description should be < 2000 characters + description: truncate(v.description || v.name, { length: 2000, omission: '...' }), + player_loc: CONFIG.WEBSERVER.URL + '/videos/embed/' + v.uuid, + thumbnail_loc: v.getThumbnailStaticPath() + } + ] + })) +} + +function getSitemapBasicUrls () { + const paths = [ + '/about/instance', + '/videos/local' + ] + + return paths.map(p => ({ url: CONFIG.WEBSERVER.URL + p })) +} diff --git a/server/controllers/index.ts b/server/controllers/index.ts index 197fa897a..a88a03c79 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts @@ -6,3 +6,4 @@ export * from './services' export * from './static' export * from './webfinger' export * from './tracker' +export * from './bots' diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index 162fe2244..9a72ee96d 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -7,12 +7,12 @@ import { extname } from 'path' import { isArray } from './custom-validators/misc' import { UserModel } from '../models/account/user' -function buildNSFWFilter (res: express.Response, paramNSFW?: string) { +function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { if (paramNSFW === 'true') return true if (paramNSFW === 'false') return false if (paramNSFW === 'both') return undefined - if (res.locals.oauth) { + if (res && res.locals.oauth) { const user: UserModel = res.locals.oauth.token.User // User does not want NSFW videos diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 7195ae6c5..6b798875c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -61,6 +61,7 @@ const OAUTH_LIFETIME = { const ROUTE_CACHE_LIFETIME = { FEEDS: '15 minutes', ROBOTS: '2 hours', + SITEMAP: '1 day', SECURITYTXT: '2 hours', NODEINFO: '10 minutes', DNT_POLICY: '1 week', diff --git a/server/models/account/account.ts b/server/models/account/account.ts index 5a237d733..a99e9b1ad 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -241,6 +241,27 @@ export class AccountModel extends Model { }) } + static listLocalsForSitemap (sort: string) { + const query = { + attributes: [ ], + offset: 0, + order: getSort(sort), + include: [ + { + attributes: [ 'preferredUsername', 'serverId' ], + model: ActorModel.unscoped(), + where: { + serverId: null + } + } + ] + } + + return AccountModel + .unscoped() + .findAll(query) + } + toFormattedJSON (): Account { const actor = this.Actor.toFormattedJSON() const account = { diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index f4586917e..86bf0461a 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -233,6 +233,27 @@ export class VideoChannelModel extends Model { }) } + static listLocalsForSitemap (sort: string) { + const query = { + attributes: [ ], + offset: 0, + order: getSort(sort), + include: [ + { + attributes: [ 'preferredUsername', 'serverId' ], + model: ActorModel.unscoped(), + where: { + serverId: null + } + } + ] + } + + return VideoChannelModel + .unscoped() + .findAll(query) + } + static searchForApi (options: { actorId: number search: string diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts index 8fab20971..b53803ee1 100644 --- a/server/tests/misc-endpoints.ts +++ b/server/tests/misc-endpoints.ts @@ -2,7 +2,18 @@ import 'mocha' import * as chai from 'chai' -import { flushTests, killallServers, makeGetRequest, runServer, ServerInfo } from './utils' +import { + addVideoChannel, + createUser, + flushTests, + killallServers, + makeGetRequest, + runServer, + ServerInfo, + setAccessTokensToServers, + uploadVideo +} from './utils' +import { VideoPrivacy } from '../../shared/models/videos' const expect = chai.expect @@ -15,6 +26,7 @@ describe('Test misc endpoints', function () { await flushTests() server = await runServer(1) + await setAccessTokensToServers([ server ]) }) describe('Test a well known endpoints', function () { @@ -93,6 +105,64 @@ describe('Test misc endpoints', function () { }) }) + describe('Test bots endpoints', function () { + + it('Should get the empty sitemap', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/sitemap.xml', + statusCodeExpected: 200 + }) + + expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') + expect(res.text).to.contain('http://localhost:9001/about/instance') + }) + + it('Should get the empty cached sitemap', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/sitemap.xml', + statusCodeExpected: 200 + }) + + expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') + expect(res.text).to.contain('http://localhost:9001/about/instance') + }) + + it('Should add videos, channel and accounts and get sitemap', async function () { + this.timeout(35000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 1', nsfw: false }) + await uploadVideo(server.url, server.accessToken, { name: 'video 2', nsfw: false }) + await uploadVideo(server.url, server.accessToken, { name: 'video 3', privacy: VideoPrivacy.PRIVATE }) + + await addVideoChannel(server.url, server.accessToken, { name: 'channel1', displayName: 'channel 1' }) + await addVideoChannel(server.url, server.accessToken, { name: 'channel2', displayName: 'channel 2' }) + + await createUser(server.url, server.accessToken, 'user1', 'password') + await createUser(server.url, server.accessToken, 'user2', 'password') + + const res = await makeGetRequest({ + url: server.url, + path: '/sitemap.xml?t=1', // avoid using cache + statusCodeExpected: 200 + }) + + expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') + expect(res.text).to.contain('http://localhost:9001/about/instance') + + expect(res.text).to.contain('') + expect(res.text).to.contain('') + expect(res.text).to.not.contain('') + + expect(res.text).to.contain('http://localhost:9001/video-channels/channel1') + expect(res.text).to.contain('http://localhost:9001/video-channels/channel2') + + expect(res.text).to.contain('http://localhost:9001/accounts/user1') + expect(res.text).to.contain('http://localhost:9001/accounts/user2') + }) + }) + after(async function () { killallServers([ server ]) }) diff --git a/yarn.lock b/yarn.lock index 1cbe6756d..8d74f9d55 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7457,6 +7457,15 @@ simple-websocket@^7.0.1: readable-stream "^2.0.5" ws "^6.0.0" +sitemap@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-2.1.0.tgz#1633cb88c196d755ad94becfb1c1bcacc6d3425a" + integrity sha512-AkfA7RDVCITQo+j5CpXsMJlZ/8ENO2NtgMHYIh+YMvex2Hao/oe3MQgNa03p0aWY6srCfUA1Q02OgiWCAiuccA== + dependencies: + lodash "^4.17.10" + url-join "^4.0.0" + xmlbuilder "^10.0.0" + slash@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" @@ -8592,6 +8601,11 @@ urix@^0.1.0: resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= +url-join@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.0.tgz#4d3340e807d3773bda9991f8305acdcc2a665d2a" + integrity sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo= + url-parse-lax@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73" @@ -9001,6 +9015,11 @@ xml@^1.0.1: resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= +xmlbuilder@^10.0.0: + version "10.1.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0" + integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg== + xmlbuilder@~9.0.1: version "9.0.7" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" -- 2.25.1