From e2600d8b261994abbbeb1ff921edaefd267fc122 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 9 Aug 2019 15:04:36 +0200 Subject: [PATCH] Add avatar to prune script --- scripts/create-import-video-file-job.ts | 2 +- scripts/create-transcoding-job.ts | 2 +- scripts/prune-storage.ts | 110 +++++++--- server/controllers/lazy-static.ts | 7 +- .../lib/files-cache/videos-preview-cache.ts | 2 +- server/models/video/thumbnail.ts | 10 + server/models/video/video.ts | 23 +- server/tests/cli/index.ts | 1 + server/tests/cli/prune-storage.ts | 199 ++++++++++++++++++ shared/extra-utils/miscs/miscs.ts | 5 + shared/extra-utils/videos/videos.ts | 6 +- 11 files changed, 323 insertions(+), 44 deletions(-) create mode 100644 server/tests/cli/prune-storage.ts diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts index c8c6c6429..2b636014a 100644 --- a/scripts/create-import-video-file-job.ts +++ b/scripts/create-import-video-file-job.ts @@ -25,7 +25,7 @@ run() async function run () { await initDatabaseModels(true) - const video = await VideoModel.loadByUUIDWithFile(program['video']) + const video = await VideoModel.loadByUUID(program['video']) if (!video) throw new Error('Video not found.') if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.') diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index 2b7cb5177..2eb872169 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts @@ -29,7 +29,7 @@ run() async function run () { await initDatabaseModels(true) - const video = await VideoModel.loadByUUIDWithFile(program['video']) + const video = await VideoModel.loadByUUID(program['video']) if (!video) throw new Error('Video not found.') const dataInput: VideoTranscodingPayload = program.resolution !== undefined diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts index 4953a7439..d6dff8247 100755 --- a/scripts/prune-storage.ts +++ b/scripts/prune-storage.ts @@ -3,9 +3,12 @@ import { join } from 'path' import { CONFIG } from '../server/initializers/config' import { VideoModel } from '../server/models/video/video' import { initDatabaseModels } from '../server/initializers' -import { remove, readdir } from 'fs-extra' +import { readdir, remove } from 'fs-extra' import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy' +import * as Bluebird from 'bluebird' import { getUUIDFromFilename } from '../server/helpers/utils' +import { ThumbnailModel } from '../server/models/video/thumbnail' +import { AvatarModel } from '../server/models/avatar/avatar' run() .then(() => process.exit(0)) @@ -17,25 +20,19 @@ run() async function run () { await initDatabaseModels(true) - const storageOnlyOwnedToPrune = [ - CONFIG.STORAGE.VIDEOS_DIR, - CONFIG.STORAGE.TORRENTS_DIR, - CONFIG.STORAGE.REDUNDANCY_DIR - ] + let toDelete: string[] = [] - const storageForAllToPrune = [ - CONFIG.STORAGE.PREVIEWS_DIR, - CONFIG.STORAGE.THUMBNAILS_DIR - ] + toDelete = toDelete.concat( + await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)), + await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)), - let toDelete: string[] = [] - for (const directory of storageOnlyOwnedToPrune) { - toDelete = toDelete.concat(await pruneDirectory(directory, true)) - } + await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist), - for (const directory of storageForAllToPrune) { - toDelete = toDelete.concat(await pruneDirectory(directory, false)) - } + await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)), + await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)), + + await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist) + ) const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t))) @@ -61,30 +58,79 @@ async function run () { } } -async function pruneDirectory (directory: string, onlyOwned = false) { +type ExistFun = (file: string) => Promise +async function pruneDirectory (directory: string, existFun: ExistFun) { const files = await readdir(directory) const toDelete: string[] = [] - for (const file of files) { + await Bluebird.map(files, async file => { + if (await existFun(file) !== true) { + toDelete.push(join(directory, file)) + } + }, { concurrency: 20 }) + + return toDelete +} + +function doesVideoExist (keepOnlyOwned: boolean) { + return async (file: string) => { const uuid = getUUIDFromFilename(file) - let video: VideoModel - let localRedundancy: boolean + const video = await VideoModel.loadByUUID(uuid) - if (uuid) { - video = await VideoModel.loadByUUIDWithFile(uuid) - localRedundancy = await VideoRedundancyModel.isLocalByVideoUUIDExists(uuid) - } + return video && (keepOnlyOwned === false || video.isOwned()) + } +} - if ( - !uuid || - !video || - (onlyOwned === true && (video.isOwned() === false && localRedundancy === false)) - ) { - toDelete.push(join(directory, file)) +function doesThumbnailExist (keepOnlyOwned: boolean) { + return async (file: string) => { + const thumbnail = await ThumbnailModel.loadByName(file) + if (!thumbnail) return false + + if (keepOnlyOwned) { + const video = await VideoModel.load(thumbnail.videoId) + if (video.isOwned() === false) return false } + + return true } +} - return toDelete +async function doesAvatarExist (file: string) { + const avatar = await AvatarModel.loadByName(file) + + return !!avatar +} + +async function doesRedundancyExist (file: string) { + const uuid = getUUIDFromFilename(file) + const video = await VideoModel.loadWithFiles(uuid) + + if (!video) return false + + const isPlaylist = file.includes('.') === false + + if (isPlaylist) { + const p = video.getHLSPlaylist() + if (!p) return false + + const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id) + return !!redundancy + } + + const resolution = parseInt(file.split('-')[5], 10) + if (isNaN(resolution)) { + console.error('Cannot prune %s because we cannot guess guess the resolution.', file) + return true + } + + const videoFile = video.getFile(resolution) + if (!videoFile) { + console.error('Cannot find file of video %s - %d', video.url, resolution) + return true + } + + const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) + return !!redundancy } async function askConfirmation () { diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts index 4285fd727..28d2f862a 100644 --- a/server/controllers/lazy-static.ts +++ b/server/controllers/lazy-static.ts @@ -49,7 +49,12 @@ async function getAvatar (req: express.Request, res: express.Response) { logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl) - await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl }) + try { + await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl }) + } catch (err) { + logger.warn('Cannot process remote avatar %s.', avatar.fileUrl, { err }) + return res.sendStatus(404) + } avatar.onDisk = true avatar.save() diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts index a68619d07..3da6bb138 100644 --- a/server/lib/files-cache/videos-preview-cache.ts +++ b/server/lib/files-cache/videos-preview-cache.ts @@ -18,7 +18,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { } async getFilePathImpl (videoUUID: string) { - const video = await VideoModel.loadByUUIDWithFile(videoUUID) + const video = await VideoModel.loadByUUID(videoUUID) if (!video) return undefined if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() } diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index cf2040cbf..f1952dcc1 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -100,6 +100,16 @@ export class ThumbnailModel extends Model { .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err)) } + static loadByName (filename: string) { + const query = { + where: { + filename + } + } + + return ThumbnailModel.findOne(query) + } + static generateDefaultPreviewName (videoUUID: string) { return videoUUID + '.jpg' } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 1321337ff..b59df397d 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -119,6 +119,7 @@ import { CONFIG } from '../../initializers/config' import { ThumbnailModel } from './thumbnail' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' import { createTorrentPromise } from '../../helpers/webtorrent' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [ @@ -1422,15 +1423,23 @@ export class VideoModel extends Model { return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(options) } - static loadWithFiles (id: number, t?: Transaction, logging?: boolean) { + static loadWithFiles (id: number | string, t?: Transaction, logging?: boolean) { + const where = buildWhereIdOrUUID(id) + + const query = { + where, + transaction: t, + logging + } + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS, ScopeNames.WITH_THUMBNAILS - ]).findByPk(id, { transaction: t, logging }) + ]).findOne(query) } - static loadByUUIDWithFile (uuid: string) { + static loadByUUID (uuid: string) { const options = { where: { uuid @@ -1754,7 +1763,7 @@ export class VideoModel extends Model { return maxBy(this.VideoFiles, file => file.resolution) } - getFile (resolution: VideoResolution) { + getFile (resolution: number) { if (Array.isArray(this.VideoFiles) === false) return undefined return this.VideoFiles.find(f => f.resolution === resolution) @@ -1893,6 +1902,12 @@ export class VideoModel extends Model { return `/api/${API_VERSION}/videos/${this.uuid}/description` } + getHLSPlaylist () { + if (!this.VideoStreamingPlaylists) return undefined + + return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + } + removeFile (videoFile: VideoFileModel, isRedundancy = false) { const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts index 5af286fe2..029cd5196 100644 --- a/server/tests/cli/index.ts +++ b/server/tests/cli/index.ts @@ -4,5 +4,6 @@ import './create-transcoding-job' import './optimize-old-videos' import './peertube' import './plugins' +import './prune-storage' import './reset-password' import './update-host' diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts new file mode 100644 index 000000000..67a5c564e --- /dev/null +++ b/server/tests/cli/prune-storage.ts @@ -0,0 +1,199 @@ +/* tslint:disable:no-unused-expression */ + +import 'mocha' +import * as chai from 'chai' +import { waitJobs } from '../../../shared/extra-utils/server/jobs' +import { + buildServerDirectory, + cleanupTests, + createVideoPlaylist, + doubleFollow, + execCLI, + flushAndRunMultipleServers, + getAccount, + getEnvCli, + ServerInfo, + setAccessTokensToServers, setDefaultVideoChannel, + updateMyAvatar, + uploadVideo, + wait +} from '../../../shared/extra-utils' +import { Account, VideoPlaylistPrivacy } from '../../../shared/models' +import { createFile, readdir } from 'fs-extra' +import * as uuidv4 from 'uuid/v4' +import { join } from 'path' +import * as request from 'supertest' + +const expect = chai.expect + +async function countFiles (internalServerNumber: number, directory: string) { + const files = await readdir(buildServerDirectory(internalServerNumber, directory)) + + return files.length +} + +async function assertNotExists (internalServerNumber: number, directory: string, substring: string) { + const files = await readdir(buildServerDirectory(internalServerNumber, directory)) + + for (const f of files) { + expect(f).to.not.contain(substring) + } +} + +async function assertCountAreOkay (servers: ServerInfo[]) { + for (const server of servers) { + const videosCount = await countFiles(server.internalServerNumber, 'videos') + expect(videosCount).to.equal(8) + + const torrentsCount = await countFiles(server.internalServerNumber, 'torrents') + expect(torrentsCount).to.equal(8) + + const previewsCount = await countFiles(server.internalServerNumber, 'previews') + expect(previewsCount).to.equal(2) + + const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails') + expect(thumbnailsCount).to.equal(6) + + const avatarsCount = await countFiles(server.internalServerNumber, 'avatars') + expect(avatarsCount).to.equal(2) + } +} + +describe('Test prune storage scripts', function () { + let servers: ServerInfo[] + const badNames: { [ directory: string ]: string[] } = {} + + before(async function () { + this.timeout(120000) + + servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } }) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + for (const server of servers) { + await uploadVideo(server.url, server.accessToken, { name: 'video 1' }) + await uploadVideo(server.url, server.accessToken, { name: 'video 2' }) + + await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' }) + + await createVideoPlaylist({ + url: server.url, + token: server.accessToken, + playlistAttrs: { + displayName: 'playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.videoChannel.id, + thumbnailfile: 'thumbnail.jpg' + } + }) + } + + await doubleFollow(servers[0], servers[1]) + + // Lazy load the remote avatar + { + const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port) + const account: Account = res.body + await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200) + } + + { + const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port) + const account: Account = res.body + await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200) + } + + await wait(1000) + + await waitJobs(servers) + }) + + it('Should have the files on the disk', async function () { + await assertCountAreOkay(servers) + }) + + it('Should create some dirty files', async function () { + for (let i = 0; i < 2; i++) { + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'videos') + + const n1 = uuidv4() + '.mp4' + const n2 = uuidv4() + '.webm' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['videos'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'torrents') + + const n1 = uuidv4() + '-240.torrent' + const n2 = uuidv4() + '-480.torrent' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['torrents'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'thumbnails') + + const n1 = uuidv4() + '.jpg' + const n2 = uuidv4() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['thumbnails'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'previews') + + const n1 = uuidv4() + '.jpg' + const n2 = uuidv4() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['previews'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'avatars') + + const n1 = uuidv4() + '.png' + const n2 = uuidv4() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['avatars'] = [ n1, n2 ] + } + } + }) + + it('Should run prune storage', async function () { + this.timeout(30000) + + const env = getEnvCli(servers[0]) + await execCLI(`echo y | ${env} npm run prune-storage`) + }) + + it('Should have removed files', async function () { + await assertCountAreOkay(servers) + + for (const directory of Object.keys(badNames)) { + for (const name of badNames[directory]) { + await assertNotExists(servers[0].internalServerNumber, directory, name) + } + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/shared/extra-utils/miscs/miscs.ts b/shared/extra-utils/miscs/miscs.ts index ae8a26c98..6b0f6d990 100644 --- a/shared/extra-utils/miscs/miscs.ts +++ b/shared/extra-utils/miscs/miscs.ts @@ -44,6 +44,10 @@ function root () { return root } +function buildServerDirectory (internalServerNumber: number, directory: string) { + return join(root(), 'test' + internalServerNumber, directory) +} + async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { const res = await request(url) .get(imagePath) @@ -105,6 +109,7 @@ async function generateHighBitrateVideo () { export { dateIsValid, wait, + buildServerDirectory, webtorrentAdd, immutableAssign, testImage, diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 1533f37ab..75f7d58d7 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -19,7 +19,7 @@ import { import * as validator from 'validator' import { VideoDetails, VideoPrivacy } from '../../models/videos' import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' -import { dateIsValid, webtorrentAdd } from '../miscs/miscs' +import { dateIsValid, webtorrentAdd, buildServerDirectory } from '../miscs/miscs' loadLanguages() @@ -308,10 +308,8 @@ async function checkVideoFilesWereRemoved ( join('redundancy', 'hls') ] ) { - const testDirectory = 'test' + serverNumber - for (const directory of directories) { - const directoryPath = join(root(), testDirectory, directory) + const directoryPath = buildServerDirectory(serverNumber, directory) const directoryExists = await pathExists(directoryPath) if (directoryExists === false) continue -- 2.25.1