# strategy: 'recently-added' # Cache recently added videos
# min_views: 10 # Having at least x views
+# Other instances that duplicate your content
+remote_redundancy:
+ videos:
+ # 'nobody': Do not accept remote redundancies
+ # 'anybody': Accept remote redundancies from anybody
+ # 'followings': Accept redundancies from instance followings
+ accept_from: 'anybody'
+
csp:
enabled: false
report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk!
# strategy: 'recently-added' # Cache recently added videos
# min_views: 10 # Having at least x views
+# Other instances that duplicate your content
+remote_redundancy:
+ videos:
+ # 'nobody': Do not accept remote redundancies
+ # 'anybody': Accept remote redundancies from anybody
+ # 'followings': Accept redundancies from instance followings
+ accept_from: 'anybody'
+
csp:
enabled: false
report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk!
import { isArray } from '../helpers/custom-validators/misc'
import { uniq } from 'lodash'
import { WEBSERVER } from './constants'
+import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
async function checkActivityPubUrls () {
const actor = await getServerActor()
return 'Videos redundancy should be an array (you must uncomment lines containing - too)'
}
+ // Remote redundancies
+ const acceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM
+ const acceptFromValues = new Set<VideoRedundancyConfigFilter>([ 'nobody', 'anybody', 'followings' ])
+ if (acceptFromValues.has(acceptFrom) === false) {
+ return 'remote_redundancy.videos.accept_from has an incorrect value'
+ }
+
// Check storage directory locations
if (isProdInstance()) {
const configStorage = config.get('storage')
'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces',
'history.videos.max_age', 'views.videos.remote.max_age',
'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
- 'theme.default'
+ 'theme.default',
+ 'remote_redundancy.videos.accept_from'
]
const requiredAlternatives = [
[ // set
import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
import * as bytes from 'bytes'
+import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
// Use a variable to reload the configuration if we need
let config: IConfig = require('config')
STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
}
},
+ REMOTE_REDUNDANCY: {
+ VIDEOS: {
+ ACCEPT_FROM: config.get<VideoRedundancyConfigFilter>('remote_redundancy.videos.accept_from')
+ }
+ },
CSP: {
ENABLED: config.get<boolean>('csp.enabled'),
REPORT_ONLY: config.get<boolean>('csp.report_only'),
import { createOrUpdateVideoPlaylist } from '../playlist'
import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../typings/models'
+import { isRedundancyAccepted } from '@server/lib/redundancy'
async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
const { activity, byActor } = options
}
async function processCreateCacheFile (activity: ActivityCreate, byActor: MActorSignature) {
+ if (await isRedundancyAccepted(activity, byActor) !== true) return
+
const cacheFile = activity.object as CacheFileObject
const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
import { createOrUpdateVideoPlaylist } from '../playlist'
import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
import { MActorSignature, MAccountIdActor } from '../../../typings/models'
+import { isRedundancyAccepted } from '@server/lib/redundancy'
async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) {
const { activity, byActor } = options
}
async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) {
+ if (await isRedundancyAccepted(activity, byActor) !== true) return
+
const cacheFileObject = activity.object as CacheFileObject
if (!isCacheFileObjectValid(cacheFileObject)) {
import { sendUndoCacheFile } from './activitypub/send'
import { Transaction } from 'sequelize'
import { getServerActor } from '../helpers/utils'
-import { MVideoRedundancyVideo } from '@server/typings/models'
+import { MActorSignature, MVideoRedundancyVideo } from '@server/typings/models'
+import { CONFIG } from '@server/initializers/config'
+import { logger } from '@server/helpers/logger'
+import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
+import { Activity } from '@shared/models'
async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) {
const serverActor = await getServerActor()
}
}
+async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) {
+ const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM
+ if (configAcceptFrom === 'nobody') {
+ logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id)
+ return false
+ }
+
+ if (configAcceptFrom === 'followings') {
+ const serverActor = await getServerActor()
+ const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id)
+
+ if (allowed !== true) {
+ logger.info('Do not accept remote redundancy %s because actor %s is not followed by our instance.', activity.id, byActor.url)
+ return false
+ }
+ }
+
+ return true
+}
+
// ---------------------------------------------------------------------------
export {
+ isRedundancyAccepted,
removeRedundanciesOfServer,
removeVideoRedundancy
}
MActorFollowSubscriptions
} from '@server/typings/models'
import { ActivityPubActorType } from '@shared/models'
+import { VideoModel } from '@server/models/video/video'
@Table({
tableName: 'actorFollow',
if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved)
}
+ static isFollowedBy (actorId: number, followerActorId: number) {
+ const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
+ const options = {
+ type: QueryTypes.SELECT as QueryTypes.SELECT,
+ bind: { actorId, followerActorId },
+ raw: true
+ }
+
+ return VideoModel.sequelize.query(query, options)
+ .then(results => results.length === 1)
+ }
+
static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Bluebird<MActorFollowActorsDefault> {
const query = {
where: {
+import './redundancy-constraints'
import './redundancy'
import './manage-redundancy'
--- /dev/null
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import * as chai from 'chai'
+import 'mocha'
+import {
+ cleanupTests,
+ flushAndRunServer,
+ follow,
+ killallServers,
+ reRunServer,
+ ServerInfo,
+ setAccessTokensToServers,
+ uploadVideo,
+ waitUntilLog
+} from '../../../../shared/extra-utils'
+import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
+import { listVideoRedundancies, updateRedundancy } from '@shared/extra-utils/server/redundancy'
+
+const expect = chai.expect
+
+describe('Test redundancy constraints', function () {
+ let remoteServer: ServerInfo
+ let localServer: ServerInfo
+ let servers: ServerInfo[]
+
+ async function getTotalRedundanciesLocalServer () {
+ const res = await listVideoRedundancies({
+ url: localServer.url,
+ accessToken: localServer.accessToken,
+ target: 'my-videos'
+ })
+
+ return res.body.total
+ }
+
+ async function getTotalRedundanciesRemoteServer () {
+ const res = await listVideoRedundancies({
+ url: remoteServer.url,
+ accessToken: remoteServer.accessToken,
+ target: 'remote-videos'
+ })
+
+ return res.body.total
+ }
+
+ before(async function () {
+ this.timeout(120000)
+
+ {
+ const config = {
+ redundancy: {
+ videos: {
+ check_interval: '1 second',
+ strategies: [
+ {
+ strategy: 'recently-added',
+ min_lifetime: '1 hour',
+ size: '100MB',
+ min_views: 0
+ }
+ ]
+ }
+ }
+ }
+ remoteServer = await flushAndRunServer(1, config)
+ }
+
+ {
+ const config = {
+ remote_redundancy: {
+ videos: {
+ accept_from: 'nobody'
+ }
+ }
+ }
+ localServer = await flushAndRunServer(2, config)
+ }
+
+ servers = [ remoteServer, localServer ]
+
+ // Get the access tokens
+ await setAccessTokensToServers(servers)
+
+ await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 1 server 2' })
+
+ await waitJobs(servers)
+
+ // Server 1 and server 2 follow each other
+ await follow(remoteServer.url, [ localServer.url ], remoteServer.accessToken)
+ await waitJobs(servers)
+ await updateRedundancy(remoteServer.url, remoteServer.accessToken, localServer.host, true)
+
+ await waitJobs(servers)
+ })
+
+ it('Should have redundancy on server 1 but not on server 2 with a nobody filter', async function () {
+ this.timeout(120000)
+
+ await waitJobs(servers)
+ await waitUntilLog(remoteServer, 'Duplicated ', 5)
+ await waitJobs(servers)
+
+ {
+ const total = await getTotalRedundanciesRemoteServer()
+ expect(total).to.equal(1)
+ }
+
+ {
+ const total = await getTotalRedundanciesLocalServer()
+ expect(total).to.equal(0)
+ }
+ })
+
+ it('Should have redundancy on server 1 and on server 2 with an anybody filter', async function () {
+ this.timeout(120000)
+
+ const config = {
+ remote_redundancy: {
+ videos: {
+ accept_from: 'anybody'
+ }
+ }
+ }
+ await killallServers([ localServer ])
+ await reRunServer(localServer, config)
+
+ await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 2 server 2' })
+
+ await waitJobs(servers)
+ await waitUntilLog(remoteServer, 'Duplicated ', 10)
+ await waitJobs(servers)
+
+ {
+ const total = await getTotalRedundanciesRemoteServer()
+ expect(total).to.equal(2)
+ }
+
+ {
+ const total = await getTotalRedundanciesLocalServer()
+ expect(total).to.equal(1)
+ }
+ })
+
+ it('Should have redundancy on server 1 but not on server 2 with a followings filter', async function () {
+ this.timeout(120000)
+
+ const config = {
+ remote_redundancy: {
+ videos: {
+ accept_from: 'followings'
+ }
+ }
+ }
+ await killallServers([ localServer ])
+ await reRunServer(localServer, config)
+
+ await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 3 server 2' })
+
+ await waitJobs(servers)
+ await waitUntilLog(remoteServer, 'Duplicated ', 15)
+ await waitJobs(servers)
+
+ {
+ const total = await getTotalRedundanciesRemoteServer()
+ expect(total).to.equal(3)
+ }
+
+ {
+ const total = await getTotalRedundanciesLocalServer()
+ expect(total).to.equal(1)
+ }
+ })
+
+ it('Should have redundancy on server 1 and on server 2 with followings filter now server 2 follows server 1', async function () {
+ this.timeout(120000)
+
+ await follow(localServer.url, [ remoteServer.url ], localServer.accessToken)
+ await waitJobs(servers)
+
+ await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 4 server 2' })
+
+ await waitJobs(servers)
+ await waitUntilLog(remoteServer, 'Duplicated ', 20)
+ await waitJobs(servers)
+
+ {
+ const total = await getTotalRedundanciesRemoteServer()
+ expect(total).to.equal(4)
+ }
+
+ {
+ const total = await getTotalRedundanciesLocalServer()
+ expect(total).to.equal(2)
+ }
+ })
+
+ after(async function () {
+ await cleanupTests(servers)
+ })
+})
--- /dev/null
+export type VideoRedundancyConfigFilter = 'nobody' | 'anybody' | 'followings'
projects / oweals / peertube.git / commitdiff