// Do not use barrels because we don't want to load all modules here (we need to initialize database first)
import { logger } from './server/helpers/logger'
-import { API_VERSION, CONFIG, CACHE } from './server/initializers/constants'
+import { API_VERSION, CONFIG, FILES_CACHE } from './server/initializers/constants'
const missed = checkMissedConfig()
if (missed.length !== 0) {
import { installApplication } from './server/initializers'
import { Emailer } from './server/lib/emailer'
import { JobQueue } from './server/lib/job-queue'
-import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache'
+import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
import {
activityPubRouter,
apiRouter,
])
// Caches initializations
- VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, CACHE.PREVIEWS.MAX_AGE)
- VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, CACHE.VIDEO_CAPTIONS.MAX_AGE)
+ VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
+ VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
// Enable Schedulers
ActorFollowScheduler.Instance.enable()
STATIC_MAX_AGE,
STATIC_PATHS
} from '../initializers'
-import { VideosPreviewCache } from '../lib/cache'
+import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
import { cacheRoute } from '../middlewares/cache'
import { asyncMiddleware, videosGetValidator } from '../middlewares'
import { VideoModel } from '../models/video/video'
-import { VideosCaptionCache } from '../lib/cache/videos-caption-cache'
import { UserModel } from '../models/account/user'
import { VideoCommentModel } from '../models/video/video-comment'
import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'
}
// Sub folders of cache directory
-const CACHE = {
+const FILES_CACHE = {
PREVIEWS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
}
}
+const CACHE = {
+ USER_TOKENS: {
+ MAX_SIZE: 10000
+ }
+}
+
const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
JOB_ATTEMPTS['email'] = 1
- CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
+ FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
ACCEPT_HEADERS,
BCRYPT_SALT_SIZE,
TRACKER_RATE_LIMITS,
- CACHE,
+ FILES_CACHE,
CONFIG,
CONSTRAINTS_FIELDS,
EMBED_SIZE,
VIDEO_TRANSCODING_FPS,
FFMPEG_NICE,
VIDEO_ABUSE_STATES,
+ CACHE,
JOB_REQUEST_TIMEOUT,
USER_PASSWORD_RESET_LIFETIME,
MEMOIZE_TTL,
import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
-import { CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
+import { FILES_CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
import { sequelizeTypescript } from './database'
import { remove, ensureDir } from 'fs-extra'
// ---------------------------------------------------------------------------
function removeCacheAndTmpDirectories () {
- const cacheDirectories = Object.keys(CACHE)
- .map(k => CACHE[k].DIRECTORY)
+ const cacheDirectories = Object.keys(FILES_CACHE)
+ .map(k => FILES_CACHE[k].DIRECTORY)
const tasks: Promise<any>[] = []
function createDirectoriesIfNotExist () {
const storage = CONFIG.STORAGE
- const cacheDirectories = Object.keys(CACHE)
- .map(k => CACHE[k].DIRECTORY)
+ const cacheDirectories = Object.keys(FILES_CACHE)
+ .map(k => FILES_CACHE[k].DIRECTORY)
const tasks: Promise<void>[] = []
for (const key of Object.keys(storage)) {
+++ /dev/null
-import * as AsyncLRU from 'async-lru'
-import { createWriteStream, remove } from 'fs-extra'
-import { logger } from '../../helpers/logger'
-import { VideoModel } from '../../models/video/video'
-import { fetchRemoteVideoStaticFile } from '../activitypub'
-
-export abstract class AbstractVideoStaticFileCache <T> {
-
- protected lru
-
- abstract getFilePath (params: T): Promise<string>
-
- // Load and save the remote file, then return the local path from filesystem
- protected abstract loadRemoteFile (key: string): Promise<string>
-
- init (max: number, maxAge: number) {
- this.lru = new AsyncLRU({
- max,
- maxAge,
- load: (key, cb) => {
- this.loadRemoteFile(key)
- .then(res => cb(null, res))
- .catch(err => cb(err))
- }
- })
-
- this.lru.on('evict', (obj: { key: string, value: string }) => {
- remove(obj.value)
- .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name))
- })
- }
-
- protected loadFromLRU (key: string) {
- return new Promise<string>((res, rej) => {
- this.lru.get(key, (err, value) => {
- err ? rej(err) : res(value)
- })
- })
- }
-
- protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) {
- return new Promise<string>((res, rej) => {
- const req = fetchRemoteVideoStaticFile(video, remoteStaticPath, rej)
-
- const stream = createWriteStream(destPath)
-
- req.pipe(stream)
- .on('error', (err) => rej(err))
- .on('finish', () => res(destPath))
- })
- }
-}
+++ /dev/null
-import { ACTOR_FOLLOW_SCORE } from '../../initializers'
-import { logger } from '../../helpers/logger'
-
-// Cache follows scores, instead of writing them too often in database
-// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores
-class ActorFollowScoreCache {
-
- private static instance: ActorFollowScoreCache
- private pendingFollowsScore: { [ url: string ]: number } = {}
-
- private constructor () {}
-
- static get Instance () {
- return this.instance || (this.instance = new this())
- }
-
- updateActorFollowsScore (goodInboxes: string[], badInboxes: string[]) {
- if (goodInboxes.length === 0 && badInboxes.length === 0) return
-
- logger.info('Updating %d good actor follows and %d bad actor follows scores in cache.', goodInboxes.length, badInboxes.length)
-
- for (const goodInbox of goodInboxes) {
- if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0
-
- this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS
- }
-
- for (const badInbox of badInboxes) {
- if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0
-
- this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY
- }
- }
-
- getPendingFollowsScoreCopy () {
- return this.pendingFollowsScore
- }
-
- clearPendingFollowsScore () {
- this.pendingFollowsScore = {}
- }
-}
-
-export {
- ActorFollowScoreCache
-}
+++ /dev/null
-export * from './actor-follow-score-cache'
-export * from './videos-preview-cache'
-export * from './videos-caption-cache'
+++ /dev/null
-import { join } from 'path'
-import { CACHE, CONFIG } from '../../initializers'
-import { VideoModel } from '../../models/video/video'
-import { VideoCaptionModel } from '../../models/video/video-caption'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
-
-type GetPathParam = { videoId: string, language: string }
-
-class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
-
- private static readonly KEY_DELIMITER = '%'
- private static instance: VideosCaptionCache
-
- private constructor () {
- super()
- }
-
- static get Instance () {
- return this.instance || (this.instance = new this())
- }
-
- async getFilePath (params: GetPathParam) {
- const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language)
- if (!videoCaption) return undefined
-
- if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName())
-
- const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language
- return this.loadFromLRU(key)
- }
-
- protected async loadRemoteFile (key: string) {
- const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER)
-
- const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language)
- if (!videoCaption) return undefined
-
- if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.')
-
- // Used to fetch the path
- const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
- if (!video) return undefined
-
- const remoteStaticPath = videoCaption.getCaptionStaticPath()
- const destPath = join(CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
-
- return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
- }
-}
-
-export {
- VideosCaptionCache
-}
+++ /dev/null
-import { join } from 'path'
-import { CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
-import { VideoModel } from '../../models/video/video'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
-
-class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
-
- private static instance: VideosPreviewCache
-
- private constructor () {
- super()
- }
-
- static get Instance () {
- return this.instance || (this.instance = new this())
- }
-
- async getFilePath (videoUUID: string) {
- const video = await VideoModel.loadByUUIDWithFile(videoUUID)
- if (!video) return undefined
-
- if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
-
- return this.loadFromLRU(videoUUID)
- }
-
- protected async loadRemoteFile (key: string) {
- const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key)
- if (!video) return undefined
-
- if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
-
- const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
- const destPath = join(CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
-
- return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
- }
-}
-
-export {
- VideosPreviewCache
-}
--- /dev/null
+import * as AsyncLRU from 'async-lru'
+import { createWriteStream, remove } from 'fs-extra'
+import { logger } from '../../helpers/logger'
+import { VideoModel } from '../../models/video/video'
+import { fetchRemoteVideoStaticFile } from '../activitypub'
+
+export abstract class AbstractVideoStaticFileCache <T> {
+
+ protected lru
+
+ abstract getFilePath (params: T): Promise<string>
+
+ // Load and save the remote file, then return the local path from filesystem
+ protected abstract loadRemoteFile (key: string): Promise<string>
+
+ init (max: number, maxAge: number) {
+ this.lru = new AsyncLRU({
+ max,
+ maxAge,
+ load: (key, cb) => {
+ this.loadRemoteFile(key)
+ .then(res => cb(null, res))
+ .catch(err => cb(err))
+ }
+ })
+
+ this.lru.on('evict', (obj: { key: string, value: string }) => {
+ remove(obj.value)
+ .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name))
+ })
+ }
+
+ protected loadFromLRU (key: string) {
+ return new Promise<string>((res, rej) => {
+ this.lru.get(key, (err, value) => {
+ err ? rej(err) : res(value)
+ })
+ })
+ }
+
+ protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) {
+ return new Promise<string>((res, rej) => {
+ const req = fetchRemoteVideoStaticFile(video, remoteStaticPath, rej)
+
+ const stream = createWriteStream(destPath)
+
+ req.pipe(stream)
+ .on('error', (err) => rej(err))
+ .on('finish', () => res(destPath))
+ })
+ }
+}
--- /dev/null
+import { ACTOR_FOLLOW_SCORE } from '../../initializers'
+import { logger } from '../../helpers/logger'
+
+// Cache follows scores, instead of writing them too often in database
+// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores
+class ActorFollowScoreCache {
+
+ private static instance: ActorFollowScoreCache
+ private pendingFollowsScore: { [ url: string ]: number } = {}
+
+ private constructor () {}
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ updateActorFollowsScore (goodInboxes: string[], badInboxes: string[]) {
+ if (goodInboxes.length === 0 && badInboxes.length === 0) return
+
+ logger.info('Updating %d good actor follows and %d bad actor follows scores in cache.', goodInboxes.length, badInboxes.length)
+
+ for (const goodInbox of goodInboxes) {
+ if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0
+
+ this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS
+ }
+
+ for (const badInbox of badInboxes) {
+ if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0
+
+ this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY
+ }
+ }
+
+ getPendingFollowsScoreCopy () {
+ return this.pendingFollowsScore
+ }
+
+ clearPendingFollowsScore () {
+ this.pendingFollowsScore = {}
+ }
+}
+
+export {
+ ActorFollowScoreCache
+}
--- /dev/null
+export * from './actor-follow-score-cache'
+export * from './videos-preview-cache'
+export * from './videos-caption-cache'
--- /dev/null
+import { join } from 'path'
+import { FILES_CACHE, CONFIG } from '../../initializers'
+import { VideoModel } from '../../models/video/video'
+import { VideoCaptionModel } from '../../models/video/video-caption'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+type GetPathParam = { videoId: string, language: string }
+
+class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
+
+ private static readonly KEY_DELIMITER = '%'
+ private static instance: VideosCaptionCache
+
+ private constructor () {
+ super()
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ async getFilePath (params: GetPathParam) {
+ const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language)
+ if (!videoCaption) return undefined
+
+ if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName())
+
+ const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language
+ return this.loadFromLRU(key)
+ }
+
+ protected async loadRemoteFile (key: string) {
+ const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER)
+
+ const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language)
+ if (!videoCaption) return undefined
+
+ if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.')
+
+ // Used to fetch the path
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
+ if (!video) return undefined
+
+ const remoteStaticPath = videoCaption.getCaptionStaticPath()
+ const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
+
+ return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
+ }
+}
+
+export {
+ VideosCaptionCache
+}
--- /dev/null
+import { join } from 'path'
+import { FILES_CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
+import { VideoModel } from '../../models/video/video'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
+
+ private static instance: VideosPreviewCache
+
+ private constructor () {
+ super()
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ async getFilePath (videoUUID: string) {
+ const video = await VideoModel.loadByUUIDWithFile(videoUUID)
+ if (!video) return undefined
+
+ if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
+
+ return this.loadFromLRU(videoUUID)
+ }
+
+ protected async loadRemoteFile (key: string) {
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key)
+ if (!video) return undefined
+
+ if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
+
+ const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
+ const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
+
+ return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
+ }
+}
+
+export {
+ VideosPreviewCache
+}
import * as Bluebird from 'bluebird'
import { logger } from '../../../helpers/logger'
import { doRequest } from '../../../helpers/requests'
-import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers'
-import { ActorFollowScoreCache } from '../../cache'
+import { ActorFollowScoreCache } from '../../files-cache'
export type ActivitypubHttpBroadcastPayload = {
uris: string[]
import { doRequest } from '../../../helpers/requests'
import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
import { JOB_REQUEST_TIMEOUT } from '../../../initializers'
-import { ActorFollowScoreCache } from '../../cache'
+import { ActorFollowScoreCache } from '../../files-cache'
export type ActivitypubHttpUnicastPayload = {
uri: string
import { UserModel } from '../models/account/user'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { OAuthTokenModel } from '../models/oauth/oauth-token'
-import { CONFIG } from '../initializers/constants'
+import { CONFIG, CACHE } from '../initializers/constants'
import { Transaction } from 'sequelize'
type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
-const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
-const userHavingToken: { [ userId: number ]: string } = {}
+let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
+let userHavingToken: { [ userId: number ]: string } = {}
// ---------------------------------------------------------------------------
return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
.then(tokenModel => {
if (tokenModel) {
+ // Reinit our cache
+ if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
+ accessTokenCache = {}
+ userHavingToken = {}
+ }
+
accessTokenCache[ bearerToken ] = tokenModel
userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
}
import { ActorFollowModel } from '../../models/activitypub/actor-follow'
import { AbstractScheduler } from './abstract-scheduler'
import { SCHEDULER_INTERVALS_MS } from '../../initializers'
-import { ActorFollowScoreCache } from '../cache'
+import { ActorFollowScoreCache } from '../files-cache'
export class ActorFollowScheduler extends AbstractScheduler {
projects / oweals / peertube.git / commitdiff