From 598edb8af1cc7e5ea3ead1ec9c96c4853b90be36 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 3 Feb 2020 11:12:29 +0100 Subject: [PATCH] Reduce AP context size on specific activities --- server/controllers/activitypub/client.ts | 2 +- server/helpers/activitypub.ts | 179 +++++++++--------- server/lib/activitypub/send/send-announce.ts | 2 +- server/lib/activitypub/send/send-view.ts | 2 +- server/lib/activitypub/send/utils.ts | 30 +-- .../handlers/activitypub-http-broadcast.ts | 2 + .../handlers/activitypub-http-unicast.ts | 2 + .../handlers/utils/activitypub-http-utils.ts | 6 +- 8 files changed, 121 insertions(+), 104 deletions(-) diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 39f6d7231..2812bfe1e 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -234,7 +234,7 @@ async function videoAnnounceController (req: express.Request, res: express.Respo const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.videoAll, undefined) - return activityPubResponse(activityPubContextify(activity), res) + return activityPubResponse(activityPubContextify(activity, 'Announce'), res) } async function videoAnnouncesController (req: express.Request, res: express.Response) { diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 22f8550ca..326785b68 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -8,93 +8,100 @@ import { pageToStartAndCount } from './core-utils' import { URL } from 'url' import { MActor, MVideoAccountLight } from '../typings/models' -function activityPubContextify (data: T) { - return Object.assign(data, { +export type ContextType = 'All' | 'View' | 'Announce' + +function activityPubContextify (data: T, type: ContextType = 'All') { + const base = { + RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' + } + + if (type === 'All') { + Object.assign(base, { + pt: 'https://joinpeertube.org/ns#', + sc: 'http://schema.org#', + Hashtag: 'as:Hashtag', + uuid: 'sc:identifier', + category: 'sc:category', + licence: 'sc:license', + subtitleLanguage: 'sc:subtitleLanguage', + sensitive: 'as:sensitive', + language: 'sc:inLanguage', + expires: 'sc:expires', + CacheFile: 'pt:CacheFile', + Infohash: 'pt:Infohash', + originallyPublishedAt: 'sc:datePublished', + views: { + '@type': 'sc:Number', + '@id': 'pt:views' + }, + state: { + '@type': 'sc:Number', + '@id': 'pt:state' + }, + size: { + '@type': 'sc:Number', + '@id': 'pt:size' + }, + fps: { + '@type': 'sc:Number', + '@id': 'pt:fps' + }, + startTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:startTimestamp' + }, + stopTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:stopTimestamp' + }, + position: { + '@type': 'sc:Number', + '@id': 'pt:position' + }, + commentsEnabled: { + '@type': 'sc:Boolean', + '@id': 'pt:commentsEnabled' + }, + downloadEnabled: { + '@type': 'sc:Boolean', + '@id': 'pt:downloadEnabled' + }, + waitTranscoding: { + '@type': 'sc:Boolean', + '@id': 'pt:waitTranscoding' + }, + support: { + '@type': 'sc:Text', + '@id': 'pt:support' + }, + likes: { + '@id': 'as:likes', + '@type': '@id' + }, + dislikes: { + '@id': 'as:dislikes', + '@type': '@id' + }, + playlists: { + '@id': 'pt:playlists', + '@type': '@id' + }, + shares: { + '@id': 'as:shares', + '@type': '@id' + }, + comments: { + '@id': 'as:comments', + '@type': '@id' + } + }) + } + + return Object.assign({}, data, { '@context': [ 'https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', - { - RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', - pt: 'https://joinpeertube.org/ns#', - sc: 'http://schema.org#', - Hashtag: 'as:Hashtag', - uuid: 'sc:identifier', - category: 'sc:category', - licence: 'sc:license', - subtitleLanguage: 'sc:subtitleLanguage', - sensitive: 'as:sensitive', - language: 'sc:inLanguage', - expires: 'sc:expires', - CacheFile: 'pt:CacheFile', - Infohash: 'pt:Infohash', - originallyPublishedAt: 'sc:datePublished', - views: { - '@type': 'sc:Number', - '@id': 'pt:views' - }, - state: { - '@type': 'sc:Number', - '@id': 'pt:state' - }, - size: { - '@type': 'sc:Number', - '@id': 'pt:size' - }, - fps: { - '@type': 'sc:Number', - '@id': 'pt:fps' - }, - startTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:startTimestamp' - }, - stopTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:stopTimestamp' - }, - position: { - '@type': 'sc:Number', - '@id': 'pt:position' - }, - commentsEnabled: { - '@type': 'sc:Boolean', - '@id': 'pt:commentsEnabled' - }, - downloadEnabled: { - '@type': 'sc:Boolean', - '@id': 'pt:downloadEnabled' - }, - waitTranscoding: { - '@type': 'sc:Boolean', - '@id': 'pt:waitTranscoding' - }, - support: { - '@type': 'sc:Text', - '@id': 'pt:support' - } - }, - { - likes: { - '@id': 'as:likes', - '@type': '@id' - }, - dislikes: { - '@id': 'as:dislikes', - '@type': '@id' - }, - playlists: { - '@id': 'pt:playlists', - '@type': '@id' - }, - shares: { - '@id': 'as:shares', - '@type': '@id' - }, - comments: { - '@id': 'as:comments', - '@type': '@id' - } - } + base ] }) } @@ -148,8 +155,8 @@ async function activityPubCollectionPagination ( } -function buildSignedActivity (byActor: MActor, data: Object) { - const activity = activityPubContextify(data) +function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) { + const activity = activityPubContextify(data, contextType) return signJsonLDObject(byActor, activity) as Promise } diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts index a0f33852c..d03b358f1 100644 --- a/server/lib/activitypub/send/send-announce.ts +++ b/server/lib/activitypub/send/send-announce.ts @@ -28,7 +28,7 @@ async function sendVideoAnnounce (byActor: MActorLight, videoShare: MVideoShare, logger.info('Creating job to send announce %s.', videoShare.url) const followersException = [ byActor ] - return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException) + return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException, 'Announce') } function buildAnnounceActivity (url: string, byActor: MActorLight, object: string, audience?: ActivityAudience): ActivityAnnounce { diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts index 8809417f9..47482b9a9 100644 --- a/server/lib/activitypub/send/send-view.ts +++ b/server/lib/activitypub/send/send-view.ts @@ -16,7 +16,7 @@ async function sendView (byActor: ActorModel, video: MVideoAccountLight, t: Tran return buildViewActivity(url, byActor, video, audience) } - return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t, contextType: 'View' }) } function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityView { diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts index 6fb4efd60..ce932eb1f 100644 --- a/server/lib/activitypub/send/utils.ts +++ b/server/lib/activitypub/send/utils.ts @@ -8,13 +8,15 @@ import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAud import { getServerActor } from '../../../helpers/utils' import { afterCommitIfTransaction } from '../../../helpers/database-utils' import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight } from '../../../typings/models' +import { ContextType } from '@server/helpers/activitypub' async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { byActor: MActorLight video: MVideoAccountLight - transaction?: Transaction + transaction?: Transaction, + contextType?: ContextType }) { - const { byActor, video, transaction } = options + const { byActor, video, transaction, contextType } = options const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) @@ -24,7 +26,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud const activity = activityBuilder(audience) return afterCommitIfTransaction(transaction, () => { - return unicastTo(activity, byActor, video.VideoChannel.Account.Actor.getSharedInbox()) + return unicastTo(activity, byActor, video.VideoChannel.Account.Actor.getSharedInbox(), contextType) }) } @@ -34,7 +36,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud const actorsException = [ byActor ] - return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, transaction, actorsException) + return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, transaction, actorsException, contextType) } async function forwardVideoRelatedActivity ( @@ -90,11 +92,12 @@ async function broadcastToFollowers ( byActor: MActorId, toFollowersOf: MActorId[], t: Transaction, - actorsException: MActorWithInboxes[] = [] + actorsException: MActorWithInboxes[] = [], + contextType?: ContextType ) { const uris = await computeFollowerUris(toFollowersOf, actorsException, t) - return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor)) + return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor, contextType)) } async function broadcastToActors ( @@ -102,13 +105,14 @@ async function broadcastToActors ( byActor: MActorId, toActors: MActor[], t?: Transaction, - actorsException: MActorWithInboxes[] = [] + actorsException: MActorWithInboxes[] = [], + contextType?: ContextType ) { const uris = await computeUris(toActors, actorsException) - return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor)) + return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor, contextType)) } -function broadcastTo (uris: string[], data: any, byActor: MActorId) { +function broadcastTo (uris: string[], data: any, byActor: MActorId, contextType?: ContextType) { if (uris.length === 0) return undefined logger.debug('Creating broadcast job.', { uris }) @@ -116,19 +120,21 @@ function broadcastTo (uris: string[], data: any, byActor: MActorId) { const payload = { uris, signatureActorId: byActor.id, - body: data + body: data, + contextType } return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }) } -function unicastTo (data: any, byActor: MActorId, toActorUrl: string) { +function unicastTo (data: any, byActor: MActorId, toActorUrl: string, contextType?: ContextType) { logger.debug('Creating unicast job.', { uri: toActorUrl }) const payload = { uri: toActorUrl, signatureActorId: byActor.id, - body: data + body: data, + contextType } JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload }) diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 0ff7b44a0..7d9dd61e9 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -5,11 +5,13 @@ import { doRequest } from '../../../helpers/requests' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers/constants' import { ActorFollowScoreCache } from '../../files-cache' +import { ContextType } from '@server/helpers/activitypub' export type ActivitypubHttpBroadcastPayload = { uris: string[] signatureActorId?: number body: any + contextType?: ContextType } async function processActivityPubHttpBroadcast (job: Bull.Job) { diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index c70ce3be9..6b71e2891 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -4,11 +4,13 @@ import { doRequest } from '../../../helpers/requests' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { JOB_REQUEST_TIMEOUT } from '../../../initializers/constants' import { ActorFollowScoreCache } from '../../files-cache' +import { ContextType } from '@server/helpers/activitypub' export type ActivitypubHttpUnicastPayload = { uri: string signatureActorId?: number body: any + contextType?: ContextType } async function processActivityPubHttpUnicast (job: Bull.Job) { diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index a28f3596f..54b35840d 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts @@ -1,11 +1,11 @@ -import { buildSignedActivity } from '../../../../helpers/activitypub' +import { buildSignedActivity, ContextType } from '../../../../helpers/activitypub' import { getServerActor } from '../../../../helpers/utils' import { ActorModel } from '../../../../models/activitypub/actor' import { sha256 } from '../../../../helpers/core-utils' import { HTTP_SIGNATURE } from '../../../../initializers/constants' import { MActor } from '../../../../typings/models' -type Payload = { body: any, signatureActorId?: number } +type Payload = { body: any, contextType?: ContextType, signatureActorId?: number } async function computeBody (payload: Payload) { let body = payload.body @@ -13,7 +13,7 @@ async function computeBody (payload: Payload) { if (payload.signatureActorId) { const actorSignature = await ActorModel.load(payload.signatureActorId) if (!actorSignature) throw new Error('Unknown signature actor id.') - body = await buildSignedActivity(actorSignature, payload.body) + body = await buildSignedActivity(actorSignature, payload.body, payload.contextType) } return body -- 2.25.1