From 09cababd79f9d445aa027c93cdfe823745fa041a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 28 Feb 2018 18:04:46 +0100 Subject: [PATCH] Add stats route --- .../edit-custom-config.component.ts | 2 +- .../+admin/config/shared/config.service.ts | 2 +- .../src/app/+admin/jobs/shared/job.service.ts | 2 +- client/src/app/core/server/server.service.ts | 2 +- server/controllers/activitypub/inbox.ts | 2 + server/controllers/api/config.ts | 4 +- server/controllers/api/server/index.ts | 2 + server/controllers/api/server/stats.ts | 39 +++++++ server/models/account/user.ts | 9 ++ server/models/activitypub/actor-follow.ts | 50 ++++----- server/models/video/video-comment.ts | 26 +++++ server/models/video/video.ts | 23 ++++ server/tests/api/check-params/config.ts | 2 +- server/tests/api/index-fast.ts | 3 +- server/tests/api/index-slow.ts | 1 - server/tests/api/server/config.ts | 4 +- server/tests/api/server/stats.ts | 102 ++++++++++++++++++ server/tests/utils/server/config.ts | 2 +- server/tests/utils/server/stats.ts | 17 +++ shared/models/index.ts | 4 +- .../models/{config => server}/about.model.ts | 0 .../{config => server}/custom-config.model.ts | 0 .../{config => server}/customization.model.ts | 0 shared/models/{ => server}/job.model.ts | 0 .../{config => server}/server-config.model.ts | 0 shared/models/server/server-stats.model.ts | 12 +++ 26 files changed, 268 insertions(+), 42 deletions(-) create mode 100644 server/controllers/api/server/stats.ts create mode 100644 server/tests/api/server/stats.ts create mode 100644 server/tests/utils/server/stats.ts rename shared/models/{config => server}/about.model.ts (100%) rename shared/models/{config => server}/custom-config.model.ts (100%) rename shared/models/{config => server}/customization.model.ts (100%) rename shared/models/{ => server}/job.model.ts (100%) rename shared/models/{config => server}/server-config.model.ts (100%) create mode 100644 shared/models/server/server-stats.model.ts diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index ccec89a8e..cf93b4060 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -13,7 +13,7 @@ import { TRANSCODING_THREADS } from '@app/shared/forms/form-validators/custom-config' import { NotificationsService } from 'angular2-notifications' -import { CustomConfig } from '../../../../../../shared/models/config/custom-config.model' +import { CustomConfig } from '../../../../../../shared/models/server/custom-config.model' @Component({ selector: 'my-edit-custom-config', diff --git a/client/src/app/+admin/config/shared/config.service.ts b/client/src/app/+admin/config/shared/config.service.ts index 13f1f6cd2..2a39c7155 100644 --- a/client/src/app/+admin/config/shared/config.service.ts +++ b/client/src/app/+admin/config/shared/config.service.ts @@ -1,6 +1,6 @@ import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' -import { CustomConfig } from '../../../../../../shared/models/config/custom-config.model' +import { CustomConfig } from '../../../../../../shared/models/server/custom-config.model' import { environment } from '../../../../environments/environment' import { RestExtractor, RestService } from '../../../shared' diff --git a/client/src/app/+admin/jobs/shared/job.service.ts b/client/src/app/+admin/jobs/shared/job.service.ts index 1c0915b5e..98f29b742 100644 --- a/client/src/app/+admin/jobs/shared/job.service.ts +++ b/client/src/app/+admin/jobs/shared/job.service.ts @@ -6,7 +6,7 @@ import 'rxjs/add/operator/map' import { Observable } from 'rxjs/Observable' import { ResultList } from '../../../../../../shared' import { JobState } from '../../../../../../shared/models' -import { Job } from '../../../../../../shared/models/job.model' +import { Job } from '../../../../../../shared/models/server/job.model' import { environment } from '../../../../environments/environment' import { RestExtractor, RestPagination, RestService } from '../../../shared' diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts index 3c94f09c6..984738948 100644 --- a/client/src/app/core/server/server.service.ts +++ b/client/src/app/core/server/server.service.ts @@ -3,7 +3,7 @@ import { Injectable } from '@angular/core' import 'rxjs/add/operator/do' import { ReplaySubject } from 'rxjs/ReplaySubject' import { ServerConfig } from '../../../../../shared' -import { About } from '../../../../../shared/models/config/about.model' +import { About } from '../../../../../shared/models/server/about.model' import { environment } from '../../../environments/environment' @Injectable() diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts index bd0d7a9c8..0354d7833 100644 --- a/server/controllers/activitypub/inbox.ts +++ b/server/controllers/activitypub/inbox.ts @@ -56,6 +56,8 @@ async function inboxController (req: express.Request, res: express.Response, nex specificActor = res.locals.videoChannel } + logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor) + await processActivities(activities, res.locals.signature.actor, specificActor) res.status(204).end() diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 7ef0c19e3..427125810 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -1,8 +1,8 @@ import * as express from 'express' import { omit } from 'lodash' import { ServerConfig, UserRight } from '../../../shared' -import { About } from '../../../shared/models/config/about.model' -import { CustomConfig } from '../../../shared/models/config/custom-config.model' +import { About } from '../../../shared/models/server/about.model' +import { CustomConfig } from '../../../shared/models/server/custom-config.model' import { unlinkPromise, writeFilePromise } from '../../helpers/core-utils' import { isSignupAllowed } from '../../helpers/utils' import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers' diff --git a/server/controllers/api/server/index.ts b/server/controllers/api/server/index.ts index 8dc1a0031..850a52cdb 100644 --- a/server/controllers/api/server/index.ts +++ b/server/controllers/api/server/index.ts @@ -1,9 +1,11 @@ import * as express from 'express' import { serverFollowsRouter } from './follows' +import { statsRouter } from './stats' const serverRouter = express.Router() serverRouter.use('/', serverFollowsRouter) +serverRouter.use('/', statsRouter) // --------------------------------------------------------------------------- diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts new file mode 100644 index 000000000..6f4fe938c --- /dev/null +++ b/server/controllers/api/server/stats.ts @@ -0,0 +1,39 @@ +import * as express from 'express' +import { ServerStats } from '../../../../shared/models/server/server-stats.model' +import { asyncMiddleware } from '../../../middlewares' +import { UserModel } from '../../../models/account/user' +import { ActorFollowModel } from '../../../models/activitypub/actor-follow' +import { VideoModel } from '../../../models/video/video' +import { VideoCommentModel } from '../../../models/video/video-comment' + +const statsRouter = express.Router() + +statsRouter.get('/stats', + asyncMiddleware(getStats) +) + +async function getStats (req: express.Request, res: express.Response, next: express.NextFunction) { + const { totalLocalVideos, totalLocalVideoViews, totalVideos } = await VideoModel.getStats() + const { totalLocalVideoComments, totalVideoComments } = await VideoCommentModel.getStats() + const { totalUsers } = await UserModel.getStats() + const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() + + const data: ServerStats = { + totalLocalVideos, + totalLocalVideoViews, + totalVideos, + totalLocalVideoComments, + totalVideoComments, + totalUsers, + totalInstanceFollowers, + totalInstanceFollowing + } + + return res.json(data).end() +} + +// --------------------------------------------------------------------------- + +export { + statsRouter +} diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 74cf0f4a8..afa9d7be0 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -13,6 +13,7 @@ import { comparePassword, cryptPassword } from '../../helpers/peertube-crypto' import { OAuthTokenModel } from '../oauth/oauth-token' import { getSort, throwIfNotValid } from '../utils' import { VideoChannelModel } from '../video/video-channel' +import { VideoCommentModel } from '../video/video-comment' import { AccountModel } from './account' @DefaultScope({ @@ -226,6 +227,14 @@ export class UserModel extends Model { }) } + static async getStats () { + const totalUsers = await UserModel.count() + + return { + totalUsers + } + } + hasRight (right: UserRight) { return hasUserRight(this.role, right) } diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 8260904a1..3c11d1b67 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts @@ -8,6 +8,7 @@ import { import { FollowState } from '../../../shared/models/actors' import { AccountFollow } from '../../../shared/models/actors/follow.model' import { logger } from '../../helpers/logger' +import { getServerActor } from '../../helpers/utils' import { ACTOR_FOLLOW_SCORE } from '../../initializers' import { FOLLOW_STATES } from '../../initializers/constants' import { ServerModel } from '../server/server' @@ -182,34 +183,6 @@ export class ActorFollowModel extends Model { return ActorFollowModel.findOne(query) } - static loadByFollowerInbox (url: string, t?: Sequelize.Transaction) { - const query = { - where: { - state: 'accepted' - }, - include: [ - { - model: ActorModel, - required: true, - as: 'ActorFollower', - where: { - [Sequelize.Op.or]: [ - { - inboxUrl: url - }, - { - sharedInboxUrl: url - } - ] - } - } - ], - transaction: t - } as any // FIXME: typings does not work - - return ActorFollowModel.findOne(query) - } - static listFollowingForApi (id: number, start: number, count: number, sort: string) { const query = { distinct: true, @@ -296,6 +269,27 @@ export class ActorFollowModel extends Model { return ActorFollowModel.createListAcceptedFollowForApiQuery('following', actorIds, t, start, count) } + static async getStats () { + const serverActor = await getServerActor() + + const totalInstanceFollowing = await ActorFollowModel.count({ + where: { + actorId: serverActor.id + } + }) + + const totalInstanceFollowers = await ActorFollowModel.count({ + where: { + targetActorId: serverActor.id + } + }) + + return { + totalInstanceFollowing, + totalInstanceFollowers + } + } + private static async createListAcceptedFollowForApiQuery ( type: 'followers' | 'following', actorIds: number[], diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index 47e3211a3..bf8da924d 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -326,6 +326,32 @@ export class VideoCommentModel extends Model { .findAll(query) } + static async getStats () { + const totalLocalVideoComments = await VideoCommentModel.count({ + include: [ + { + model: AccountModel, + required: true, + include: [ + { + model: ActorModel, + required: true, + where: { + serverId: null + } + } + ] + } + ] + }) + const totalVideoComments = await VideoCommentModel.count() + + return { + totalLocalVideoComments, + totalVideoComments + } + } + getThreadId (): number { return this.originCommentId || this.id } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 80ca513bf..f6a21814c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -761,6 +761,29 @@ export class VideoModel extends Model { .findOne(options) } + static async getStats () { + const totalLocalVideos = await VideoModel.count({ + where: { + remote: false + } + }) + const totalVideos = await VideoModel.count() + + let totalLocalVideoViews = await VideoModel.sum('views', { + where: { + remote: false + } + }) + // Sequelize could return null... + if (!totalLocalVideoViews) totalLocalVideoViews = 0 + + return { + totalLocalVideos, + totalLocalVideoViews, + totalVideos + } + } + getOriginalFile () { if (Array.isArray(this.VideoFiles) === false) return undefined diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index c1c0a3f59..a66e51a6a 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -2,7 +2,7 @@ import { omit } from 'lodash' import 'mocha' -import { CustomConfig } from '../../../../shared/models/config/custom-config.model' +import { CustomConfig } from '../../../../shared/models/server/custom-config.model' import { createUser, flushTests, killallServers, makeDeleteRequest, makeGetRequest, makePutBodyRequest, runServer, ServerInfo, diff --git a/server/tests/api/index-fast.ts b/server/tests/api/index-fast.ts index 9f52310dd..464dcb5e0 100644 --- a/server/tests/api/index-fast.ts +++ b/server/tests/api/index-fast.ts @@ -1,5 +1,5 @@ // Order of the tests we want to execute -import './server/config' +import './server/stats' import './check-params' import './users/users' import './videos/single-server' @@ -10,3 +10,4 @@ import './videos/video-description' import './videos/video-privacy' import './videos/services' import './server/email' +import './server/config' diff --git a/server/tests/api/index-slow.ts b/server/tests/api/index-slow.ts index 0082bcb56..cde546856 100644 --- a/server/tests/api/index-slow.ts +++ b/server/tests/api/index-slow.ts @@ -1,5 +1,4 @@ // Order of the tests we want to execute -// import './multiple-servers' import './videos/video-transcoder' import './videos/multiple-servers' import './server/follows' diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 048135a34..3d90580d8 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -2,8 +2,8 @@ import 'mocha' import * as chai from 'chai' -import { About } from '../../../../shared/models/config/about.model' -import { CustomConfig } from '../../../../shared/models/config/custom-config.model' +import { About } from '../../../../shared/models/server/about.model' +import { CustomConfig } from '../../../../shared/models/server/custom-config.model' import { deleteCustomConfig, getAbout, killallServers, reRunServer } from '../../utils' const expect = chai.expect diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts new file mode 100644 index 000000000..71d54c0ab --- /dev/null +++ b/server/tests/api/server/stats.ts @@ -0,0 +1,102 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { ServerStats } from '../../../../shared/models/server/server-stats.model' +import { + createUser, + doubleFollow, + flushAndRunMultipleServers, + follow, + killallServers, + ServerInfo, + uploadVideo, + viewVideo, + wait +} from '../../utils' +import { flushTests, setAccessTokensToServers } from '../../utils/index' +import { getStats } from '../../utils/server/stats' +import { addVideoCommentThread } from '../../utils/videos/video-comments' + +const expect = chai.expect + +describe('Test stats', function () { + let servers: ServerInfo[] = [] + + before(async function () { + this.timeout(60000) + + await flushTests() + servers = await flushAndRunMultipleServers(3) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + const user = { + username: 'user1', + password: 'super_password' + } + await createUser(servers[0].url, servers[0].accessToken, user.username, user.password) + + const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, {}) + const videoUUID = resVideo.body.video.uuid + + await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment') + + await viewVideo(servers[0].url, videoUUID) + + await follow(servers[2].url, [ servers[0].url ], servers[2].accessToken) + await wait(5000) + }) + + it('Should have the correct stats on instance 1', async function () { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.totalLocalVideoComments).to.equal(1) + expect(data.totalLocalVideos).to.equal(1) + expect(data.totalLocalVideoViews).to.equal(1) + expect(data.totalUsers).to.equal(2) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(2) + expect(data.totalInstanceFollowing).to.equal(1) + }) + + it('Should have the correct stats on instance 2', async function () { + const res = await getStats(servers[1].url) + const data: ServerStats = res.body + + expect(data.totalLocalVideoComments).to.equal(0) + expect(data.totalLocalVideos).to.equal(0) + expect(data.totalLocalVideoViews).to.equal(0) + expect(data.totalUsers).to.equal(1) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(1) + expect(data.totalInstanceFollowing).to.equal(1) + }) + + it('Should have the correct stats on instance 3', async function () { + const res = await getStats(servers[2].url) + const data: ServerStats = res.body + + expect(data.totalLocalVideoComments).to.equal(0) + expect(data.totalLocalVideos).to.equal(0) + expect(data.totalLocalVideoViews).to.equal(0) + expect(data.totalUsers).to.equal(1) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(0) + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/utils/server/config.ts b/server/tests/utils/server/config.ts index e5411117a..57f95a603 100644 --- a/server/tests/utils/server/config.ts +++ b/server/tests/utils/server/config.ts @@ -1,5 +1,5 @@ import { makeDeleteRequest, makeGetRequest, makePutBodyRequest } from '../' -import { CustomConfig } from '../../../../shared/models/config/custom-config.model' +import { CustomConfig } from '../../../../shared/models/server/custom-config.model' function getConfig (url: string) { const path = '/api/v1/config' diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts new file mode 100644 index 000000000..9cdec6cff --- /dev/null +++ b/server/tests/utils/server/stats.ts @@ -0,0 +1,17 @@ +import { makeGetRequest } from '../' + +function getStats (url: string) { + const path = '/api/v1/server/stats' + + return makeGetRequest({ + url, + path, + statusCodeExpected: 200 + }) +} + +// --------------------------------------------------------------------------- + +export { + getStats +} diff --git a/shared/models/index.ts b/shared/models/index.ts index 1b877774c..ae3a44777 100644 --- a/shared/models/index.ts +++ b/shared/models/index.ts @@ -2,7 +2,7 @@ export * from './actors' export * from './activitypub' export * from './users' export * from './videos' -export * from './job.model' +export * from './server/job.model' export * from './oauth-client-local.model' export * from './result-list.model' -export * from './config/server-config.model' +export * from './server/server-config.model' diff --git a/shared/models/config/about.model.ts b/shared/models/server/about.model.ts similarity index 100% rename from shared/models/config/about.model.ts rename to shared/models/server/about.model.ts diff --git a/shared/models/config/custom-config.model.ts b/shared/models/server/custom-config.model.ts similarity index 100% rename from shared/models/config/custom-config.model.ts rename to shared/models/server/custom-config.model.ts diff --git a/shared/models/config/customization.model.ts b/shared/models/server/customization.model.ts similarity index 100% rename from shared/models/config/customization.model.ts rename to shared/models/server/customization.model.ts diff --git a/shared/models/job.model.ts b/shared/models/server/job.model.ts similarity index 100% rename from shared/models/job.model.ts rename to shared/models/server/job.model.ts diff --git a/shared/models/config/server-config.model.ts b/shared/models/server/server-config.model.ts similarity index 100% rename from shared/models/config/server-config.model.ts rename to shared/models/server/server-config.model.ts diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts new file mode 100644 index 000000000..5c1bf3468 --- /dev/null +++ b/shared/models/server/server-stats.model.ts @@ -0,0 +1,12 @@ +export interface ServerStats { + totalUsers: number + totalLocalVideos: number + totalLocalVideoViews: number + totalLocalVideoComments: number + + totalVideos: number + totalVideoComments: number + + totalInstanceFollowers: number + totalInstanceFollowing: number +} -- 2.25.1