// ---------------------------------------------------------------------------
+const buildSamples = memoizee(async function () {
+ const [ categories, channels, tags ] = await Promise.all([
+ VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+ VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+ TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
+ ])
+
+ return { categories, channels, tags }
+}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
+
// This endpoint could be quite long, but we cache it
async function getVideosOverview (req: express.Request, res: express.Response) {
const attributes = await buildSamples()
return res.json(result)
}
-const buildSamples = memoizee(async function () {
- const [ categories, channels, tags ] = await Promise.all([
- VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
- VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
- TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
- ])
-
- return { categories, channels, tags }
-}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
-
async function getVideosByTag (tag: string, res: express.Response) {
const videos = await getVideos(res, { tagsOneOf: [ tag ] })
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { VideoModel } from '../../../models/video/video'
import { VideoCommentModel } from '../../../models/video/video-comment'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
+import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
+import { cacheRoute } from '../../../middlewares/cache'
const statsRouter = express.Router()
statsRouter.get('/stats',
+ asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)),
asyncMiddleware(getStats)
)
const { totalUsers } = await UserModel.getStats()
const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
+ const videosRedundancyStats = await Promise.all(
+ CONFIG.REDUNDANCY.VIDEOS.map(r => {
+ return VideoRedundancyModel.getStats(r.strategy)
+ .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
+ })
+ )
+
const data: ServerStats = {
totalLocalVideos,
totalLocalVideoViews,
totalVideoComments,
totalUsers,
totalInstanceFollowers,
- totalInstanceFollowing
+ totalInstanceFollowing,
+ videosRedundancy: videosRedundancyStats
}
return res.json(data).end()
},
ACTIVITY_PUB: {
VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
- }
+ },
+ STATS: '4 hours'
}
// ---------------------------------------------------------------------------
.findAll(query)
}
+ static async getStats (strategy: VideoRedundancyStrategy) {
+ const actor = await getServerActor()
+
+ const query = {
+ raw: true,
+ attributes: [
+ [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
+ [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ],
+ [ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ]
+ ],
+ where: {
+ strategy,
+ actorId: actor.id
+ },
+ include: [
+ {
+ attributes: [],
+ model: VideoFileModel,
+ required: true
+ }
+ ]
+ }
+
+ return VideoRedundancyModel.find(query as any) // FIXME: typings
+ .then((r: any) => ({
+ totalUsed: parseInt(r.totalUsed.toString(), 10),
+ totalVideos: r.totalVideos,
+ totalVideoFiles: r.totalVideoFiles
+ }))
+ }
+
toActivityPubObject (): CacheFileObject {
return {
id: this.url,
import { readdir } from 'fs-extra'
import { join } from 'path'
import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
+import { getStats } from '../../utils/server/stats'
+import { ServerStats } from '../../../../shared/models/server/server-stats.model'
const expect = chai.expect
await waitJobs(servers)
}
-async function check1WebSeed () {
+async function check1WebSeed (strategy: VideoRedundancyStrategy) {
const webseeds = [
'http://localhost:9002/static/webseed/' + video1Server2UUID
]
for (const server of servers) {
- const res = await getVideo(server.url, video1Server2UUID)
+ {
+ const res = await getVideo(server.url, video1Server2UUID)
- const video: VideoDetails = res.body
- video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+ const video: VideoDetails = res.body
+ video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+ }
+
+ {
+ const res = await getStats(server.url)
+ const data: ServerStats = res.body
+
+ expect(data.videosRedundancy).to.have.lengthOf(1)
+
+ const stat = data.videosRedundancy[0]
+ expect(stat.strategy).to.equal(strategy)
+ expect(stat.totalSize).to.equal(102400)
+ expect(stat.totalUsed).to.equal(0)
+ expect(stat.totalVideoFiles).to.equal(0)
+ expect(stat.totalVideos).to.equal(0)
+ }
}
}
expect(server2.following.hostRedundancyAllowed).to.be.true
}
-async function check2Webseeds () {
+async function check2Webseeds (strategy: VideoRedundancyStrategy) {
await waitJobs(servers)
await wait(15000)
await waitJobs(servers)
]
for (const server of servers) {
- const res = await getVideo(server.url, video1Server2UUID)
+ {
+ const res = await getVideo(server.url, video1Server2UUID)
- const video: VideoDetails = res.body
+ const video: VideoDetails = res.body
- for (const file of video.files) {
- checkMagnetWebseeds(file, webseeds)
+ for (const file of video.files) {
+ checkMagnetWebseeds(file, webseeds)
+ }
}
}
for (const resolution of [ 240, 360, 480, 720 ]) {
expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
}
+
+ {
+ const res = await getStats(servers[0].url)
+ const data: ServerStats = res.body
+
+ expect(data.videosRedundancy).to.have.lengthOf(1)
+ const stat = data.videosRedundancy[0]
+
+ expect(stat.strategy).to.equal(strategy)
+ expect(stat.totalSize).to.equal(102400)
+ expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
+ expect(stat.totalVideoFiles).to.equal(4)
+ expect(stat.totalVideos).to.equal(1)
+ }
}
async function cleanServers () {
describe('Test videos redundancy', function () {
describe('With most-views strategy', function () {
+ const strategy = 'most-views'
before(function () {
this.timeout(120000)
- return runServers('most-views')
+ return runServers(strategy)
})
it('Should have 1 webseed on the first video', function () {
- return check1WebSeed()
+ return check1WebSeed(strategy)
})
it('Should enable redundancy on server 1', function () {
it('Should have 2 webseed on the first video', function () {
this.timeout(40000)
- return check2Webseeds()
+ return check2Webseeds(strategy)
})
after(function () {
})
describe('With trending strategy', function () {
+ const strategy = 'trending'
before(function () {
this.timeout(120000)
- return runServers('trending')
+ return runServers(strategy)
})
it('Should have 1 webseed on the first video', function () {
- return check1WebSeed()
+ return check1WebSeed(strategy)
})
it('Should enable redundancy on server 1', function () {
it('Should have 2 webseed on the first video', function () {
this.timeout(40000)
- return check2Webseeds()
+ return check2Webseeds(strategy)
})
after(function () {
})
describe('With recently added strategy', function () {
+ const strategy = 'recently-added'
before(function () {
this.timeout(120000)
- return runServers('recently-added', { minViews: 3 })
+ return runServers(strategy, { minViews: 3 })
})
it('Should have 1 webseed on the first video', function () {
- return check1WebSeed()
+ return check1WebSeed(strategy)
})
it('Should enable redundancy on server 1', function () {
await wait(15000)
await waitJobs(servers)
- return check1WebSeed()
+ return check1WebSeed(strategy)
})
it('Should view 2 times the first video', async function () {
it('Should have 2 webseed on the first video', function () {
this.timeout(40000)
- return check2Webseeds()
+ return check2Webseeds(strategy)
})
after(function () {
const expect = chai.expect
-describe('Test stats', function () {
+describe('Test stats (excluding redundancy)', function () {
let servers: ServerInfo[] = []
before(async function () {
expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(2)
expect(data.totalInstanceFollowing).to.equal(1)
+ expect(data.videosRedundancy).to.have.lengthOf(0)
})
it('Should have the correct stats on instance 2', async function () {
expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(1)
expect(data.totalInstanceFollowing).to.equal(1)
+ expect(data.videosRedundancy).to.have.lengthOf(0)
})
it('Should have the correct stats on instance 3', async function () {
expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowing).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(0)
+ expect(data.videosRedundancy).to.have.lengthOf(0)
})
after(async function () {
import { makeGetRequest } from '../'
-function getStats (url: string) {
+function getStats (url: string, useCache = false) {
const path = '/api/v1/server/stats'
+ const query = {
+ t: useCache ? undefined : new Date().getTime()
+ }
+
return makeGetRequest({
url,
path,
+ query,
statusCodeExpected: 200
})
}
+import { VideoRedundancyStrategy } from '../redundancy'
+
export interface ServerStats {
totalUsers: number
totalLocalVideos: number
totalInstanceFollowers: number
totalInstanceFollowing: number
+
+ videosRedundancy: {
+ strategy: VideoRedundancyStrategy
+ totalSize: number
+ totalUsed: number
+ totalVideoFiles: number
+ totalVideos: number
+ }[]
}
projects / oweals / peertube.git / commitdiff