From 557b13ae24019d9ab214bbea7eaa0f892c8f4b05 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 9 Aug 2019 11:32:40 +0200 Subject: [PATCH] Lazy load avatars --- package.json | 2 + server.ts | 2 + server/controllers/index.ts | 1 + server/controllers/lazy-static.ts | 80 +++++++++++++++++++ server/controllers/static.ts | 24 +----- server/initializers/constants.ts | 22 ++++- .../migrations/0420-avatar-lazy.ts | 60 ++++++++++++++ server/lib/activitypub/actor.ts | 67 +++++++++------- .../lib/activitypub/process/process-update.ts | 10 ++- server/lib/avatar.ts | 38 ++++++++- server/lib/oauth-model.ts | 34 ++++---- server/models/account/user-notification.ts | 4 +- server/models/activitypub/actor.ts | 2 +- server/models/avatar/avatar.ts | 43 ++++++++-- server/models/video/thumbnail.ts | 4 +- server/models/video/video-caption.ts | 4 +- server/models/video/video.ts | 3 +- yarn.lock | 14 +++- 18 files changed, 323 insertions(+), 91 deletions(-) create mode 100644 server/controllers/lazy-static.ts create mode 100644 server/initializers/migrations/0420-avatar-lazy.ts diff --git a/package.json b/package.json index e8821bc70..481e6a3d9 100644 --- a/package.json +++ b/package.json @@ -130,6 +130,7 @@ "jsonld": "~1.1.0", "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017", "lodash": "^4.17.10", + "lru-cache": "^5.1.1", "magnet-uri": "^5.1.4", "memoizee": "^0.4.14", "morgan": "^1.5.3", @@ -179,6 +180,7 @@ "@types/fs-extra": "^8.0.0", "@types/libxmljs": "^0.18.0", "@types/lodash": "^4.14.64", + "@types/lru-cache": "^5.1.0", "@types/magnet-uri": "^5.1.1", "@types/maildev": "^0.0.1", "@types/memoizee": "^0.4.2", diff --git a/server.ts b/server.ts index 6896add34..50511a906 100644 --- a/server.ts +++ b/server.ts @@ -97,6 +97,7 @@ import { clientsRouter, feedsRouter, staticRouter, + lazyStaticRouter, servicesRouter, pluginsRouter, webfingerRouter, @@ -192,6 +193,7 @@ app.use('/', botsRouter) // Static files app.use('/', staticRouter) +app.use('/', lazyStaticRouter) // Client files, last valid routes! if (cli.client) app.use('/', clientsRouter) diff --git a/server/controllers/index.ts b/server/controllers/index.ts index 8b3501712..0d64b33bb 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts @@ -4,6 +4,7 @@ export * from './client' export * from './feeds' export * from './services' export * from './static' +export * from './lazy-static' export * from './webfinger' export * from './tracker' export * from './bots' diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts new file mode 100644 index 000000000..4285fd727 --- /dev/null +++ b/server/controllers/lazy-static.ts @@ -0,0 +1,80 @@ +import * as cors from 'cors' +import * as express from 'express' +import { LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' +import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' +import { asyncMiddleware } from '../middlewares' +import { AvatarModel } from '../models/avatar/avatar' +import { logger } from '../helpers/logger' +import { avatarPathUnsafeCache, pushAvatarProcessInQueue } from '../lib/avatar' + +const lazyStaticRouter = express.Router() + +lazyStaticRouter.use(cors()) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.AVATARS + ':filename', + asyncMiddleware(getAvatar) +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.PREVIEWS + ':uuid.jpg', + asyncMiddleware(getPreview) +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt', + asyncMiddleware(getVideoCaption) +) + +// --------------------------------------------------------------------------- + +export { + lazyStaticRouter, + getPreview, + getVideoCaption +} + +// --------------------------------------------------------------------------- + +async function getAvatar (req: express.Request, res: express.Response) { + const filename = req.params.filename + + if (avatarPathUnsafeCache.has(filename)) { + return res.sendFile(avatarPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) + } + + const avatar = await AvatarModel.loadByName(filename) + if (avatar.onDisk === false) { + if (!avatar.fileUrl) return res.sendStatus(404) + + logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl) + + await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl }) + + avatar.onDisk = true + avatar.save() + .catch(err => logger.error('Cannot save new avatar disk state.', { err })) + } + + const path = avatar.getPath() + + avatarPathUnsafeCache.set(filename, path) + return res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER }) +} + +async function getPreview (req: express.Request, res: express.Response) { + const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid) + if (!result) return res.sendStatus(404) + + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER }) +} + +async function getVideoCaption (req: express.Request, res: express.Response) { + const result = await VideosCaptionCache.Instance.getFilePath({ + videoId: req.params.videoId, + language: req.params.captionLanguage + }) + if (!result) return res.sendStatus(404) + + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER }) +} diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 110d25031..8979ef5f3 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -9,7 +9,6 @@ import { STATIC_PATHS, WEBSERVER } from '../initializers/constants' -import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' import { cacheRoute } from '../middlewares/cache' import { asyncMiddleware, videosGetValidator } from '../middlewares' import { VideoModel } from '../models/video/video' @@ -19,6 +18,7 @@ import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/node import { join } from 'path' import { root } from '../helpers/core-utils' import { CONFIG } from '../initializers/config' +import { getPreview, getVideoCaption } from './lazy-static' const staticRouter = express.Router() @@ -72,19 +72,20 @@ staticRouter.use( express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist ) +// DEPRECATED: use lazy-static route instead const avatarsPhysicalPath = CONFIG.STORAGE.AVATARS_DIR staticRouter.use( STATIC_PATHS.AVATARS, express.static(avatarsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist ) -// We don't have video previews, fetch them from the origin instance +// DEPRECATED: use lazy-static route instead staticRouter.use( STATIC_PATHS.PREVIEWS + ':uuid.jpg', asyncMiddleware(getPreview) ) -// We don't have video captions, fetch them from the origin instance +// DEPRECATED: use lazy-static route instead staticRouter.use( STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt', asyncMiddleware(getVideoCaption) @@ -177,23 +178,6 @@ export { // --------------------------------------------------------------------------- -async function getPreview (req: express.Request, res: express.Response) { - const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid) - if (!result) return res.sendStatus(404) - - return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE }) -} - -async function getVideoCaption (req: express.Request, res: express.Response) { - const result = await VideosCaptionCache.Instance.getFilePath({ - videoId: req.params.videoId, - language: req.params.captionLanguage - }) - if (!result) return res.sendStatus(404) - - return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE }) -} - async function generateNodeinfo (req: express.Request, res: express.Response) { const { totalVideos } = await VideoModel.getStats() const { totalLocalVideoComments } = await VideoCommentModel.getStats() diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index b9d90b2bd..3dc178b11 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 415 +const LAST_MIGRATION_VERSION = 420 // --------------------------------------------------------------------------- @@ -498,6 +498,11 @@ const STATIC_DOWNLOAD_PATHS = { TORRENTS: '/download/torrents/', VIDEOS: '/download/videos/' } +const LAZY_STATIC_PATHS = { + AVATARS: '/lazy-static/avatars/', + PREVIEWS: '/static/previews/', + VIDEO_CAPTIONS: '/static/video-captions/' +} // Cache control let STATIC_MAX_AGE = { @@ -536,9 +541,12 @@ const FILES_CACHE = { } } -const CACHE = { +const LRU_CACHE = { USER_TOKENS: { - MAX_SIZE: 10000 + MAX_SIZE: 1000 + }, + AVATAR_STATIC: { + MAX_SIZE: 500 } } @@ -549,6 +557,10 @@ const MEMOIZE_TTL = { OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours } +const QUEUE_CONCURRENCY = { + AVATAR_PROCESS_IMAGE: 3 +} + const REDUNDANCY = { VIDEOS: { RANDOMIZED_FACTOR: 5 @@ -649,6 +661,7 @@ export { WEBSERVER, API_VERSION, PEERTUBE_VERSION, + LAZY_STATIC_PATHS, HLS_REDUNDANCY_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, AVATARS_SIZE, @@ -695,11 +708,12 @@ export { VIDEO_PRIVACIES, VIDEO_LICENCES, VIDEO_STATES, + QUEUE_CONCURRENCY, VIDEO_RATE_TYPES, VIDEO_TRANSCODING_FPS, FFMPEG_NICE, VIDEO_ABUSE_STATES, - CACHE, + LRU_CACHE, JOB_REQUEST_TIMEOUT, USER_PASSWORD_RESET_LIFETIME, MEMOIZE_TTL, diff --git a/server/initializers/migrations/0420-avatar-lazy.ts b/server/initializers/migrations/0420-avatar-lazy.ts new file mode 100644 index 000000000..5fc57aac2 --- /dev/null +++ b/server/initializers/migrations/0420-avatar-lazy.ts @@ -0,0 +1,60 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + { + // We'll add a unique index on filename, so delete duplicates or PeerTube won't start + const query = 'DELETE FROM "avatar" s1 ' + + 'USING (SELECT MIN(id) as id, filename FROM "avatar" GROUP BY "filename" HAVING COUNT(*) > 1) s2 ' + + 'WHERE s1."filename" = s2."filename" AND s1.id <> s2.id' + await utils.sequelize.query(query) + } + + { + const data = { + type: Sequelize.STRING, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.addColumn('avatar', 'fileUrl', data) + } + + { + const data = { + type: Sequelize.BOOLEAN, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.addColumn('avatar', 'onDisk', data) + } + + { + const query = 'UPDATE "avatar" SET "onDisk" = true;' + await utils.sequelize.query(query) + } + + { + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: null + } + + await utils.queryInterface.changeColumn('avatar', 'onDisk', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 04296864b..9f5d12eb4 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -10,9 +10,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doRequest, downloadImage } from '../../helpers/requests' +import { doRequest } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants' +import { MIMETYPES, WEBSERVER } from '../../initializers/constants' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' @@ -21,7 +21,6 @@ import { VideoChannelModel } from '../../models/video/video-channel' import { JobQueue } from '../job-queue' import { getServerActor } from '../../helpers/utils' import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' -import { CONFIG } from '../../initializers/config' import { sequelizeTypescript } from '../../initializers/database' // Set account keys, this could be long so process after the account creation and do not block the client @@ -141,25 +140,27 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ actorInstance.followingUrl = attributes.following } -async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) { - if (avatarName !== undefined) { - if (actorInstance.avatarId) { +async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) { + if (info.name !== undefined) { + if (actor.avatarId) { try { - await actorInstance.Avatar.destroy({ transaction: t }) + await actor.Avatar.destroy({ transaction: t }) } catch (err) { - logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err }) + logger.error('Cannot remove old avatar of actor %s.', actor.url, { err }) } } const avatar = await AvatarModel.create({ - filename: avatarName + filename: info.name, + onDisk: info.onDisk, + fileUrl: info.fileUrl }, { transaction: t }) - actorInstance.set('avatarId', avatar.id) - actorInstance.Avatar = avatar + actor.avatarId = avatar.id + actor.Avatar = avatar } - return actorInstance + return actor } async function fetchActorTotalItems (url: string) { @@ -179,17 +180,17 @@ async function fetchActorTotalItems (url: string) { } } -async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { +async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) { if ( actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && isActivityPubUrlValid(actorJSON.icon.url) ) { const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] - const avatarName = uuidv4() + extension - await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) - - return avatarName + return { + name: uuidv4() + extension, + fileUrl: actorJSON.icon.url + } } return undefined @@ -245,8 +246,14 @@ async function refreshActorIfNeeded ( return sequelizeTypescript.transaction(async t => { updateInstanceWithAnother(actor, result.actor) - if (result.avatarName !== undefined) { - await updateActorAvatarInstance(actor, result.avatarName, t) + if (result.avatar !== undefined) { + const avatarInfo = { + name: result.avatar.name, + fileUrl: result.avatar.fileUrl, + onDisk: false + } + + await updateActorAvatarInstance(actor, avatarInfo, t) } // Force update @@ -279,7 +286,7 @@ export { buildActorInstance, setAsyncActorKeys, fetchActorTotalItems, - fetchAvatarIfExists, + getAvatarInfoIfExists, updateActorInstance, refreshActorIfNeeded, updateActorAvatarInstance, @@ -314,14 +321,17 @@ function saveActorAndServerAndModelIfNotExist ( const [ server ] = await ServerModel.findOrCreate(serverOptions) // Save our new account in database - actor.set('serverId', server.id) + actor.serverId = server.id // Avatar? - if (result.avatarName) { + if (result.avatar) { const avatar = await AvatarModel.create({ - filename: result.avatarName + filename: result.avatar.name, + fileUrl: result.avatar.fileUrl, + onDisk: false }, { transaction: t }) - actor.set('avatarId', avatar.id) + + actor.avatarId = avatar.id } // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists @@ -355,7 +365,10 @@ type FetchRemoteActorResult = { summary: string support?: string playlists?: string - avatarName?: string + avatar?: { + name: string, + fileUrl: string + } attributedTo: ActivityPubAttributedTo[] } async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { @@ -399,7 +412,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe followingUrl: actorJSON.following }) - const avatarName = await fetchAvatarIfExists(actorJSON) + const avatarInfo = await getAvatarInfoIfExists(actorJSON) const name = actorJSON.name || actorJSON.preferredUsername return { @@ -407,7 +420,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe result: { actor, name, - avatarName, + avatar: avatarInfo, summary: actorJSON.summary, support: actorJSON.support, playlists: actorJSON.playlists, diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index e3c862221..414f9e375 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -6,7 +6,7 @@ import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' import { VideoChannelModel } from '../../../models/video/video-channel' -import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' +import { getAvatarInfoIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' @@ -105,7 +105,7 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) let accountOrChannelFieldsSave: object // Fetch icon? - const avatarName = await fetchAvatarIfExists(actorAttributesToUpdate) + const avatarInfo = await getAvatarInfoIfExists(actorAttributesToUpdate) try { await sequelizeTypescript.transaction(async t => { @@ -118,8 +118,10 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) await updateActorInstance(actor, actorAttributesToUpdate) - if (avatarName !== undefined) { - await updateActorAvatarInstance(actor, avatarName, t) + if (avatarInfo !== undefined) { + const avatarOptions = Object.assign({}, avatarInfo, { onDisk: false }) + + await updateActorAvatarInstance(actor, avatarOptions, t) } await actor.save({ transaction: t }) diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 09b4e38ca..1b38e6cb5 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts @@ -1,6 +1,6 @@ import 'multer' import { sendUpdateActor } from './activitypub/send' -import { AVATARS_SIZE } from '../initializers/constants' +import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants' import { updateActorAvatarInstance } from './activitypub' import { processImage } from '../helpers/image-utils' import { AccountModel } from '../models/account/account' @@ -10,6 +10,9 @@ import { retryTransactionWrapper } from '../helpers/database-utils' import * as uuidv4 from 'uuid/v4' import { CONFIG } from '../initializers/config' import { sequelizeTypescript } from '../initializers/database' +import * as LRUCache from 'lru-cache' +import { queue } from 'async' +import { downloadImage } from '../helpers/requests' async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { const extension = extname(avatarPhysicalFile.filename) @@ -19,7 +22,13 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a return retryTransactionWrapper(() => { return sequelizeTypescript.transaction(async t => { - const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t) + const avatarInfo = { + name: avatarName, + fileUrl: null, + onDisk: true + } + + const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarInfo, t) await updatedActor.save({ transaction: t }) await sendUpdateActor(accountOrChannel, t) @@ -29,6 +38,29 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a }) } +type DownloadImageQueueTask = { fileUrl: string, filename: string } + +const downloadImageQueue = queue((task, cb) => { + downloadImage(task.fileUrl, CONFIG.STORAGE.AVATARS_DIR, task.filename, AVATARS_SIZE) + .then(() => cb()) + .catch(err => cb(err)) +}, QUEUE_CONCURRENCY.AVATAR_PROCESS_IMAGE) + +function pushAvatarProcessInQueue (task: DownloadImageQueueTask) { + return new Promise((res, rej) => { + downloadImageQueue.push(task, err => { + if (err) return rej(err) + + return res() + }) + }) +} + +// Unsafe so could returns paths that does not exist anymore +const avatarPathUnsafeCache = new LRUCache({ max: LRU_CACHE.AVATAR_STATIC.MAX_SIZE }) + export { - updateActorAvatarFile + avatarPathUnsafeCache, + updateActorAvatarFile, + pushAvatarProcessInQueue } diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 45ac3e7c4..a1153e88a 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts @@ -4,13 +4,15 @@ import { logger } from '../helpers/logger' import { UserModel } from '../models/account/user' import { OAuthClientModel } from '../models/oauth/oauth-client' import { OAuthTokenModel } from '../models/oauth/oauth-token' -import { CACHE } from '../initializers/constants' +import { LRU_CACHE } from '../initializers/constants' import { Transaction } from 'sequelize' import { CONFIG } from '../initializers/config' +import * as LRUCache from 'lru-cache' type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } -let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} -let userHavingToken: { [ userId: number ]: string } = {} + +const accessTokenCache = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) +const userHavingToken = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) // --------------------------------------------------------------------------- @@ -21,18 +23,20 @@ function deleteUserToken (userId: number, t?: Transaction) { } function clearCacheByUserId (userId: number) { - const token = userHavingToken[userId] + const token = userHavingToken.get(userId) + if (token !== undefined) { - accessTokenCache[ token ] = undefined - userHavingToken[ userId ] = undefined + accessTokenCache.del(token) + userHavingToken.del(userId) } } function clearCacheByToken (token: string) { - const tokenModel = accessTokenCache[ token ] + const tokenModel = accessTokenCache.get(token) + if (tokenModel !== undefined) { - userHavingToken[tokenModel.userId] = undefined - accessTokenCache[ token ] = undefined + userHavingToken.del(tokenModel.userId) + accessTokenCache.del(token) } } @@ -41,19 +45,13 @@ function getAccessToken (bearerToken: string) { if (!bearerToken) return Bluebird.resolve(undefined) - if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) + if (accessTokenCache.has(bearerToken)) return Bluebird.resolve(accessTokenCache.get(bearerToken)) return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) .then(tokenModel => { if (tokenModel) { - // Reinit our cache - if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) { - accessTokenCache = {} - userHavingToken = {} - } - - accessTokenCache[ bearerToken ] = tokenModel - userHavingToken[ tokenModel.userId ] = tokenModel.accessToken + accessTokenCache.set(bearerToken, tokenModel) + userHavingToken.set(tokenModel.userId, tokenModel.accessToken) } return tokenModel diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts index a4f97037b..f38cd7e78 100644 --- a/server/models/account/user-notification.ts +++ b/server/models/account/user-notification.ts @@ -410,7 +410,7 @@ export class UserNotificationModel extends Model { id: this.ActorFollow.ActorFollower.Account.id, displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), name: this.ActorFollow.ActorFollower.preferredUsername, - avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined, + avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getStaticPath() } : undefined, host: this.ActorFollow.ActorFollower.getHost() }, following: { @@ -446,7 +446,7 @@ export class UserNotificationModel extends Model { private formatActor (accountOrChannel: AccountModel | VideoChannelModel) { const avatar = accountOrChannel.Actor.Avatar - ? { path: accountOrChannel.Actor.Avatar.getWebserverPath() } + ? { path: accountOrChannel.Actor.Avatar.getStaticPath() } : undefined return { diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index bd6a2c8fd..9cc53f78a 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -513,7 +513,7 @@ export class ActorModel extends Model { getAvatarUrl () { if (!this.avatarId) return undefined - return WEBSERVER.URL + this.Avatar.getWebserverPath() + return WEBSERVER.URL + this.Avatar.getStaticPath() } isOutdated () { diff --git a/server/models/avatar/avatar.ts b/server/models/avatar/avatar.ts index aaf1b8bd9..7a370bcd3 100644 --- a/server/models/avatar/avatar.ts +++ b/server/models/avatar/avatar.ts @@ -1,13 +1,21 @@ import { join } from 'path' -import { AfterDestroy, AllowNull, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AfterDestroy, AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { Avatar } from '../../../shared/models/avatars/avatar.model' -import { STATIC_PATHS } from '../../initializers/constants' +import { LAZY_STATIC_PATHS } from '../../initializers/constants' import { logger } from '../../helpers/logger' import { remove } from 'fs-extra' import { CONFIG } from '../../initializers/config' +import { throwIfNotValid } from '../utils' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' @Table({ - tableName: 'avatar' + tableName: 'avatar', + indexes: [ + { + fields: [ 'filename' ], + unique: true + } + ] }) export class AvatarModel extends Model { @@ -15,6 +23,15 @@ export class AvatarModel extends Model { @Column filename: string + @AllowNull(true) + @Is('AvatarFileUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'fileUrl')) + @Column + fileUrl: string + + @AllowNull(false) + @Column + onDisk: boolean + @CreatedAt createdAt: Date @@ -30,16 +47,30 @@ export class AvatarModel extends Model { .catch(err => logger.error('Cannot remove avatar file %s.', instance.filename, err)) } + static loadByName (filename: string) { + const query = { + where: { + filename + } + } + + return AvatarModel.findOne(query) + } + toFormattedJSON (): Avatar { return { - path: this.getWebserverPath(), + path: this.getStaticPath(), createdAt: this.createdAt, updatedAt: this.updatedAt } } - getWebserverPath () { - return join(STATIC_PATHS.AVATARS, this.filename) + getStaticPath () { + return join(LAZY_STATIC_PATHS.AVATARS, this.filename) + } + + getPath () { + return join(CONFIG.STORAGE.AVATARS_DIR, this.filename) } removeAvatar () { diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index b767a6874..cf2040cbf 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -1,6 +1,6 @@ import { join } from 'path' import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { STATIC_PATHS, WEBSERVER } from '../../initializers/constants' +import { LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants' import { logger } from '../../helpers/logger' import { remove } from 'fs-extra' import { CONFIG } from '../../initializers/config' @@ -87,7 +87,7 @@ export class ThumbnailModel extends Model { [ThumbnailType.PREVIEW]: { label: 'preview', directory: CONFIG.STORAGE.PREVIEWS_DIR, - staticPath: STATIC_PATHS.PREVIEWS + staticPath: LAZY_STATIC_PATHS.PREVIEWS } } diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index 76243bf48..a01565851 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -16,7 +16,7 @@ import { buildWhereIdOrUUID, throwIfNotValid } from '../utils' import { VideoModel } from './video' import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' -import { STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants' +import { LAZY_STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants' import { join } from 'path' import { logger } from '../../helpers/logger' import { remove } from 'fs-extra' @@ -163,7 +163,7 @@ export class VideoCaptionModel extends Model { } getCaptionStaticPath () { - return join(STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName()) + return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName()) } getCaptionName () { diff --git a/server/models/video/video.ts b/server/models/video/video.ts index ae29cf286..1321337ff 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -63,6 +63,7 @@ import { CONSTRAINTS_FIELDS, HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, + LAZY_STATIC_PATHS, REMOTE_SCHEME, STATIC_DOWNLOAD_PATHS, STATIC_PATHS, @@ -1856,7 +1857,7 @@ export class VideoModel extends Model { if (!preview) return null // We use a local cache, so specify our cache endpoint instead of potential remote URL - return join(STATIC_PATHS.PREVIEWS, preview.filename) + return join(LAZY_STATIC_PATHS.PREVIEWS, preview.filename) } toFormattedJSON (options?: VideoFormattingJSONOptions): Video { diff --git a/yarn.lock b/yarn.lock index 894d5e81b..0e02938a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -197,6 +197,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.136.tgz#413e85089046b865d960c9ff1d400e04c31ab60f" integrity sha512-0GJhzBdvsW2RUccNHOBkabI8HZVdOXmXbXhuKlDEd5Vv12P7oAVGfomGp3Ne21o5D/qu1WmthlNKFaoZJJeErA== +"@types/lru-cache@^5.1.0": + version "5.1.0" + resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" + integrity sha512-RaE0B+14ToE4l6UqdarKPnXwVDuigfFv+5j9Dze/Nqr23yyuqdNvzcZi3xB+3Agvi5R4EOgAksfv3lXX4vBt9w== + "@types/magnet-uri@*", "@types/magnet-uri@^5.1.1": version "5.1.2" resolved "https://registry.yarnpkg.com/@types/magnet-uri/-/magnet-uri-5.1.2.tgz#7860417399d52ddc0be1021d570b4ac93ffc133e" @@ -4394,6 +4399,13 @@ lru-cache@4.1.x, lru-cache@^4.0.1: pseudomap "^1.0.2" yallist "^2.1.2" +lru-cache@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" + integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== + dependencies: + yallist "^3.0.2" + lru-queue@0.1: version "0.1.0" resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3" @@ -8082,7 +8094,7 @@ yallist@^2.1.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= -yallist@^3.0.0, yallist@^3.0.3: +yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.3.tgz#b4b049e314be545e3ce802236d6cd22cd91c3de9" integrity sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A== -- 2.25.1