From 4195cd2bc5046d4cdf1c677c27cd41f427d7a13a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 17 Apr 2018 14:01:06 +0200 Subject: [PATCH] Add redis cache to feed route --- server/controllers/feeds.ts | 23 +++++-------- server/initializers/constants.ts | 8 +++++ server/lib/redis.ts | 57 ++++++++++++++++++++++++++++++-- server/middlewares/cache.ts | 41 +++++++++++++++++++++++ 4 files changed, 113 insertions(+), 16 deletions(-) create mode 100644 server/middlewares/cache.ts diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts index 700c50ec8..3e384c48a 100644 --- a/server/controllers/feeds.ts +++ b/server/controllers/feeds.ts @@ -1,16 +1,11 @@ import * as express from 'express' -import { CONFIG } from '../initializers' -import { - asyncMiddleware, - feedsValidator, - setDefaultPagination, - setDefaultSort, - videosSortValidator -} from '../middlewares' +import { CONFIG, FEEDS } from '../initializers/constants' +import { asyncMiddleware, feedsValidator, setDefaultSort, videosSortValidator } from '../middlewares' import { VideoModel } from '../models/video/video' import * as Feed from 'pfeed' import { ResultList } from '../../shared/models' import { AccountModel } from '../models/account/account' +import { cacheRoute } from '../middlewares/cache' const feedsRouter = express.Router() @@ -18,6 +13,7 @@ feedsRouter.get('/feeds/videos.:format', videosSortValidator, setDefaultSort, asyncMiddleware(feedsValidator), + asyncMiddleware(cacheRoute), asyncMiddleware(generateFeed) ) @@ -31,8 +27,7 @@ export { async function generateFeed (req: express.Request, res: express.Response, next: express.NextFunction) { let feed = initFeed() - const paginationStart = 0 - const paginationCount = 20 + const start = 0 let resultList: ResultList const account: AccountModel = res.locals.account @@ -40,15 +35,15 @@ async function generateFeed (req: express.Request, res: express.Response, next: if (account) { resultList = await VideoModel.listAccountVideosForApi( account.id, - paginationStart, - paginationCount, + start, + FEEDS.COUNT, req.query.sort, true ) } else { resultList = await VideoModel.listForApi( - paginationStart, - paginationCount, + start, + FEEDS.COUNT, req.query.sort, req.query.filter, true diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 56d39529e..9fde989c5 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -423,6 +423,13 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '' // --------------------------------------------------------------------------- +const FEEDS = { + COUNT: 20, + CACHE_LIFETIME: 1000 * 60 * 15 // 15 minutes +} + +// --------------------------------------------------------------------------- + // Special constants for a test instance if (isTestInstance() === true) { ACTOR_FOLLOW_SCORE.BASE = 20 @@ -462,6 +469,7 @@ export { SERVER_ACTOR_NAME, PRIVATE_RSA_KEY_SIZE, SORTABLE_COLUMNS, + FEEDS, STATIC_MAX_AGE, STATIC_PATHS, ACTIVITY_PUB, diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 41f4c9869..1e7c0a821 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -1,7 +1,14 @@ +import * as express from 'express' import { createClient, RedisClient } from 'redis' import { logger } from '../helpers/logger' import { generateRandomString } from '../helpers/utils' -import { CONFIG, USER_PASSWORD_RESET_LIFETIME, VIDEO_VIEW_LIFETIME } from '../initializers' +import { CONFIG, FEEDS, USER_PASSWORD_RESET_LIFETIME, VIDEO_VIEW_LIFETIME } from '../initializers' + +type CachedRoute = { + body: string, + contentType?: string + statusCode?: string +} class Redis { @@ -54,6 +61,22 @@ class Redis { return this.exists(this.buildViewKey(ip, videoUUID)) } + async getCachedRoute (req: express.Request) { + const cached = await this.getObject(this.buildCachedRouteKey(req)) + + return cached as CachedRoute + } + + setCachedRoute (req: express.Request, body: any, contentType?: string, statusCode?: number) { + const cached: CachedRoute = { + body: body.toString(), + contentType, + statusCode: statusCode.toString() + } + + return this.setObject(this.buildCachedRouteKey(req), cached, FEEDS.CACHE_LIFETIME) + } + listJobs (jobsPrefix: string, state: string, mode: 'alpha', order: 'ASC' | 'DESC', offset: number, count: number) { return new Promise((res, rej) => { this.client.sort(jobsPrefix + ':jobs:' + state, 'by', mode, order, 'LIMIT', offset.toString(), count.toString(), (err, values) => { @@ -79,13 +102,39 @@ class Redis { this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => { if (err) return rej(err) - if (ok !== 'OK') return rej(new Error('Redis result is not OK.')) + if (ok !== 'OK') return rej(new Error('Redis set result is not OK.')) return res() }) }) } + private setObject (key: string, obj: { [ id: string ]: string }, expirationMilliseconds: number) { + return new Promise((res, rej) => { + this.client.hmset(this.prefix + key, obj, (err, ok) => { + if (err) return rej(err) + if (!ok) return rej(new Error('Redis mset result is not OK.')) + + this.client.pexpire(this.prefix + key, expirationMilliseconds, (err, ok) => { + if (err) return rej(err) + if (!ok) return rej(new Error('Redis expiration result is not OK.')) + + return res() + }) + }) + }) + } + + private getObject (key: string) { + return new Promise<{ [ id: string ]: string }>((res, rej) => { + this.client.hgetall(this.prefix + key, (err, value) => { + if (err) return rej(err) + + return res(value) + }) + }) + } + private exists (key: string) { return new Promise((res, rej) => { this.client.exists(this.prefix + key, (err, existsNumber) => { @@ -104,6 +153,10 @@ class Redis { return videoUUID + '-' + ip } + private buildCachedRouteKey (req: express.Request) { + return req.method + '-' + req.originalUrl + } + static get Instance () { return this.instance || (this.instance = new this()) } diff --git a/server/middlewares/cache.ts b/server/middlewares/cache.ts new file mode 100644 index 000000000..a2c7f7cbd --- /dev/null +++ b/server/middlewares/cache.ts @@ -0,0 +1,41 @@ +import * as express from 'express' +import { Redis } from '../lib/redis' +import { logger } from '../helpers/logger' + +async function cacheRoute (req: express.Request, res: express.Response, next: express.NextFunction) { + const cached = await Redis.Instance.getCachedRoute(req) + + // Not cached + if (!cached) { + logger.debug('Not cached result for route %s.', req.originalUrl) + + const sendSave = res.send.bind(res) + + res.send = (body) => { + if (res.statusCode >= 200 && res.statusCode < 400) { + Redis.Instance.setCachedRoute(req, body, res.getHeader('content-type').toString(), res.statusCode) + .catch(err => logger.error('Cannot cache route.', { err })) + } + + return sendSave(body) + } + + return next() + } + + if (cached.contentType) res.contentType(cached.contentType) + + if (cached.statusCode) { + const statusCode = parseInt(cached.statusCode, 10) + if (!isNaN(statusCode)) res.status(statusCode) + } + + logger.debug('Use cached result for %s.', req.originalUrl) + return res.send(cached.body).end() +} + +// --------------------------------------------------------------------------- + +export { + cacheRoute +} -- 2.25.1