sortField="createdAt" (onLazyLoad)="loadLazy($event)"
>
<p-column field="id" header="ID" [style]="{ width: '60px' }"></p-column>
+ <p-column field="score" header="Score"></p-column>
<p-column field="follower.host" header="Host"></p-column>
- <p-column field="follower.score" header="Score"></p-column>
<p-column field="state" header="State"></p-column>
<p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
</p-dataTable>
const videoChannels = user.videoChannels
if (Array.isArray(videoChannels) === false) return
- videoChannels.forEach(c => channel.push({ id: c.id, label: c.name }))
+ videoChannels.forEach(c => channel.push({ id: c.id, label: c.displayName }))
return res()
}
[validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
></my-video-edit>
-
<div class="submit-container">
<div *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div>
</div>
<div class="video-info-channel">
- {{ video.channel.name }}
+ {{ video.channel.displayName }}
<!-- Here will be the subscribe button -->
</div>
import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs'
import { VideosPreviewCache } from './server/lib/cache'
import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers'
+import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler'
// ----------- Command line -----------
// ----------- Make the server listening -----------
server.listen(port, () => {
VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
+ BadActorFollowScheduler.Instance.enable()
+
activitypubHttpJobScheduler.activate()
transcodingJobScheduler.activate()
// ---------------------------------------------------------------------------
-const LAST_MIGRATION_VERSION = 165
+const LAST_MIGRATION_VERSION = 170
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
-// Number of points we add/remove from a friend after a successful/bad request
-const SERVERS_SCORE = {
+// Number of points we add/remove after a successful/bad request
+const ACTOR_FOLLOW_SCORE = {
PENALTY: -10,
BONUS: 10,
- BASE: 100,
- MAX: 1000
+ BASE: 1000,
+ MAX: 10000
}
const FOLLOW_STATES: { [ id: string ]: FollowState } = {
// 1 minutes
let JOBS_FETCHING_INTERVAL = 60000
+// 1 hour
+let SCHEDULER_INTERVAL = 60000 * 60
+
// ---------------------------------------------------------------------------
const CONFIG = {
// Special constants for a test instance
if (isTestInstance() === true) {
- SERVERS_SCORE.BASE = 20
+ ACTOR_FOLLOW_SCORE.BASE = 20
JOBS_FETCHING_INTERVAL = 1000
REMOTE_SCHEME.HTTP = 'http'
REMOTE_SCHEME.WS = 'ws'
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute
CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
+ SCHEDULER_INTERVAL = 10000
}
CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT)
OAUTH_LIFETIME,
OPENGRAPH_AND_OEMBED_COMMENT,
PAGINATION_COUNT_DEFAULT,
- SERVERS_SCORE,
+ ACTOR_FOLLOW_SCORE,
PREVIEWS_SIZE,
REMOTE_SCHEME,
FOLLOW_STATES,
VIDEO_LICENCES,
VIDEO_RATE_TYPES,
VIDEO_MIMETYPE_EXT,
- AVATAR_MIMETYPE_EXT
+ AVATAR_MIMETYPE_EXT,
+ SCHEDULER_INTERVAL
}
--- /dev/null
+import * as Sequelize from 'sequelize'
+import { ACTOR_FOLLOW_SCORE } from '../index'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction,
+ queryInterface: Sequelize.QueryInterface,
+ sequelize: Sequelize.Sequelize
+}): Promise<void> {
+ await utils.queryInterface.removeColumn('server', 'score')
+
+ const data = {
+ type: Sequelize.INTEGER,
+ allowNull: false,
+ defaultValue: ACTOR_FOLLOW_SCORE.BASE
+ }
+
+ await utils.queryInterface.addColumn('actorFollow', 'score', data)
+
+}
+
+function down (options) {
+ throw new Error('Not implemented.')
+}
+
+export {
+ up,
+ down
+}
import { logger } from '../../../helpers/logger'
import { doRequest } from '../../../helpers/requests'
+import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler'
async function process (payload: ActivityPubHttpPayload, jobId: number) {
httpSignature: httpSignatureOptions
}
+ const badUrls: string[] = []
+ const goodUrls: string[] = []
+
for (const uri of payload.uris) {
options.uri = uri
try {
await doRequest(options)
+ goodUrls.push(uri)
} catch (err) {
- await maybeRetryRequestLater(err, payload, uri)
+ const isRetryingLater = await maybeRetryRequestLater(err, payload, uri)
+ if (isRetryingLater === false) badUrls.push(uri)
}
}
+
+ return ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes(goodUrls, badUrls, undefined)
}
function onError (err: Error, jobId: number) {
import { getServerActor } from '../../../helpers/utils'
import { ACTIVITY_PUB } from '../../../initializers'
import { ActorModel } from '../../../models/activitypub/actor'
+import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { JobHandler, JobScheduler } from '../job-scheduler'
import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler'
const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers)
-function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) {
+async function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) {
logger.warn('Cannot make request to %s.', uri, err)
let attemptNumber = payload.attemptNumber || 1
uris: [ uri ],
attemptNumber
})
- return activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload)
+ await activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload)
+
+ return true
}
+
+ return false
}
async function computeBody (payload: ActivityPubHttpPayload) {
import { logger } from '../../../helpers/logger'
import { doRequest } from '../../../helpers/requests'
+import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler'
async function process (payload: ActivityPubHttpPayload, jobId: number) {
try {
await doRequest(options)
+ await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([ uri ], [], undefined)
} catch (err) {
- await maybeRetryRequestLater(err, payload, uri)
+ const isRetryingLater = await maybeRetryRequestLater(err, payload, uri)
+ if (isRetryingLater === false) {
+ await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([], [ uri ], undefined)
+ }
+
throw err
}
}
--- /dev/null
+import { SCHEDULER_INTERVAL } from '../../initializers'
+
+export abstract class AbstractScheduler {
+
+ private interval: NodeJS.Timer
+
+ enable () {
+ this.interval = setInterval(() => this.execute(), SCHEDULER_INTERVAL)
+ }
+
+ disable () {
+ clearInterval(this.interval)
+ }
+
+ protected abstract execute ()
+}
--- /dev/null
+import { logger } from '../../helpers/logger'
+import { ActorFollowModel } from '../../models/activitypub/actor-follow'
+import { AbstractScheduler } from './abstract-scheduler'
+
+export class BadActorFollowScheduler extends AbstractScheduler {
+
+ private static instance: AbstractScheduler
+
+ private constructor () {
+ super()
+ }
+
+ async execute () {
+ try {
+ await ActorFollowModel.removeBadActorFollows()
+ } catch (err) {
+ logger.error('Error in bad actor follows scheduler.', err)
+ }
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+}
const actor = this.Actor.toFormattedJSON()
const account = {
id: this.id,
- name: this.Actor.preferredUsername,
displayName: this.name,
createdAt: this.createdAt,
updatedAt: this.updatedAt
import * as Bluebird from 'bluebird'
import { values } from 'lodash'
import * as Sequelize from 'sequelize'
-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import {
+ AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, IsInt, Max, Model, Table,
+ UpdatedAt
+} from 'sequelize-typescript'
import { FollowState } from '../../../shared/models/actors'
+import { AccountFollow } from '../../../shared/models/actors/follow.model'
+import { logger } from '../../helpers/logger'
+import { ACTOR_FOLLOW_SCORE } from '../../initializers'
import { FOLLOW_STATES } from '../../initializers/constants'
import { ServerModel } from '../server/server'
import { getSort } from '../utils'
{
fields: [ 'actorId', 'targetActorId' ],
unique: true
+ },
+ {
+ fields: [ 'score' ]
}
]
})
@Column(DataType.ENUM(values(FOLLOW_STATES)))
state: FollowState
+ @AllowNull(false)
+ @Default(ACTOR_FOLLOW_SCORE.BASE)
+ @IsInt
+ @Max(ACTOR_FOLLOW_SCORE.MAX)
+ @Column
+ score: number
+
@CreatedAt
createdAt: Date
})
ActorFollowing: ActorModel
+ // Remove actor follows with a score of 0 (too many requests where they were unreachable)
+ static async removeBadActorFollows () {
+ const actorFollows = await ActorFollowModel.listBadActorFollows()
+
+ const actorFollowsRemovePromises = actorFollows.map(actorFollow => actorFollow.destroy())
+ await Promise.all(actorFollowsRemovePromises)
+
+ const numberOfActorFollowsRemoved = actorFollows.length
+
+ if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved)
+ }
+
+ static updateActorFollowsScoreAndRemoveBadOnes (goodInboxes: string[], badInboxes: string[], t: Sequelize.Transaction) {
+ if (goodInboxes.length === 0 && badInboxes.length === 0) return
+
+ logger.info('Updating %d good actor follows and %d bad actor follows scores.', goodInboxes.length, badInboxes.length)
+
+ if (goodInboxes.length !== 0) {
+ ActorFollowModel.incrementScores(goodInboxes, ACTOR_FOLLOW_SCORE.BONUS, t)
+ .catch(err => logger.error('Cannot increment scores of good actor follows.', err))
+ }
+
+ if (badInboxes.length !== 0) {
+ ActorFollowModel.incrementScores(badInboxes, ACTOR_FOLLOW_SCORE.PENALTY, t)
+ .catch(err => logger.error('Cannot decrement scores of bad actor follows.', err))
+ }
+ }
+
static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Sequelize.Transaction) {
const query = {
where: {
}
}
- toFormattedJSON () {
+ private static incrementScores (inboxUrls: string[], value: number, t: Sequelize.Transaction) {
+ const inboxUrlsString = inboxUrls.map(url => `'${url}'`).join(',')
+
+ const query = 'UPDATE "actorFollow" SET "score" = "score" +' + value + ' ' +
+ 'WHERE id IN (' +
+ 'SELECT "actorFollow"."id" FROM "actorFollow" ' +
+ 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."actorId" ' +
+ 'WHERE "actor"."inboxUrl" IN (' + inboxUrlsString + ') OR "actor"."sharedInboxUrl" IN (' + inboxUrlsString + ')' +
+ ')'
+
+ const options = {
+ type: Sequelize.QueryTypes.BULKUPDATE,
+ transaction: t
+ }
+
+ return ActorFollowModel.sequelize.query(query, options)
+ }
+
+ private static listBadActorFollows () {
+ const query = {
+ where: {
+ score: {
+ [Sequelize.Op.lte]: 0
+ }
+ }
+ }
+
+ return ActorFollowModel.findAll(query)
+ }
+
+ toFormattedJSON (): AccountFollow {
const follower = this.ActorFollower.toFormattedJSON()
const following = this.ActorFollowing.toFormattedJSON()
id: this.id,
follower,
following,
+ score: this.score,
state: this.state,
createdAt: this.createdAt,
updatedAt: this.updatedAt
VideoChannel: VideoChannelModel
static load (id: number) {
- return ActorModel.scope(ScopeNames.FULL).findById(id)
+ return ActorModel.unscoped().findById(id)
}
static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) {
avatar = this.Avatar.toFormattedJSON()
}
- let score: number
- if (this.Server) {
- score = this.Server.score
- }
-
return {
id: this.id,
url: this.url,
uuid: this.uuid,
+ name: this.preferredUsername,
host: this.getHost(),
- score,
followingCount: this.followingCount,
followersCount: this.followersCount,
- avatar
+ avatar,
+ createdAt: this.createdAt,
+ updatedAt: this.updatedAt
}
}
-import * as Sequelize from 'sequelize'
-import { AllowNull, Column, CreatedAt, Default, Is, IsInt, Max, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { isHostValid } from '../../helpers/custom-validators/servers'
-import { logger } from '../../helpers/logger'
-import { SERVERS_SCORE } from '../../initializers'
import { throwIfNotValid } from '../utils'
@Table({
{
fields: [ 'host' ],
unique: true
- },
- {
- fields: [ 'score' ]
}
]
})
@Column
host: string
- @AllowNull(false)
- @Default(SERVERS_SCORE.BASE)
- @IsInt
- @Max(SERVERS_SCORE.MAX)
- @Column
- score: number
-
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
-
- static updateServersScoreAndRemoveBadOnes (goodServers: number[], badServers: number[]) {
- logger.info('Updating %d good servers and %d bad servers scores.', goodServers.length, badServers.length)
-
- if (goodServers.length !== 0) {
- ServerModel.incrementScores(goodServers, SERVERS_SCORE.BONUS)
- .catch(err => {
- logger.error('Cannot increment scores of good servers.', err)
- })
- }
-
- if (badServers.length !== 0) {
- ServerModel.incrementScores(badServers, SERVERS_SCORE.PENALTY)
- .then(() => ServerModel.removeBadServers())
- .catch(err => {
- if (err) logger.error('Cannot decrement scores of bad servers.', err)
- })
-
- }
- }
-
- // Remove servers with a score of 0 (too many requests where they were unreachable)
- private static async removeBadServers () {
- try {
- const servers = await ServerModel.listBadServers()
-
- const serversRemovePromises = servers.map(server => server.destroy())
- await Promise.all(serversRemovePromises)
-
- const numberOfServersRemoved = servers.length
-
- if (numberOfServersRemoved) {
- logger.info('Removed %d servers.', numberOfServersRemoved)
- } else {
- logger.info('No need to remove bad servers.')
- }
- } catch (err) {
- logger.error('Cannot remove bad servers.', err)
- }
- }
-
- private static incrementScores (ids: number[], value: number) {
- const update = {
- score: Sequelize.literal('score +' + value)
- }
-
- const options = {
- where: {
- id: {
- [Sequelize.Op.in]: ids
- }
- },
- // In this case score is a literal and not an integer so we do not validate it
- validate: false
- }
-
- return ServerModel.update(update, options)
- }
-
- private static listBadServers () {
- const query = {
- where: {
- score: {
- [Sequelize.Op.lte]: 0
- }
- }
- }
-
- return ServerModel.findAll(query)
- }
}
const actor = this.Actor.toFormattedJSON()
const account = {
id: this.id,
- name: this.name,
+ displayName: this.name,
description: this.description,
isLocal: this.Actor.isOwned(),
createdAt: this.createdAt,
-import { Avatar } from '../avatars/avatar.model'
+import { Actor } from './actor.model'
-export interface Account {
- id: number
- uuid: string
- url: string
- name: string
+export interface Account extends Actor {
displayName: string
- host: string
- followingCount: number
- followersCount: number
- createdAt: Date
- updatedAt: Date
- avatar: Avatar
}
--- /dev/null
+import { Avatar } from '../avatars/avatar.model'
+
+export interface Actor {
+ id: number
+ uuid: string
+ url: string
+ name: string
+ host: string
+ followingCount: number
+ followersCount: number
+ createdAt: Date
+ updatedAt: Date
+ avatar: Avatar
+}
-import { Account } from './account.model'
+import { Actor } from './actor.model'
export type FollowState = 'pending' | 'accepted'
export interface AccountFollow {
id: number
- follower: Account
- following: Account
+ follower: Actor
+ following: Actor
+ score: number
state: FollowState
createdAt: Date
updatedAt: Date
+import { Actor } from '../actors/actor.model'
import { Video } from './video.model'
-export interface VideoChannel {
- id: number
- name: string
- url: string
+export interface VideoChannel extends Actor {
+ displayName: string
description: string
isLocal: boolean
- createdAt: Date | string
- updatedAt: Date | string
owner?: {
name: string
uuid: string
projects / oweals / peertube.git / commitdiff