async function run () {
await initDatabaseModels(true)
- const video = await VideoModel.loadByUUIDWithFile(program['video'])
+ const video = await VideoModel.loadByUUID(program['video'])
if (!video) throw new Error('Video not found.')
if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.')
async function run () {
await initDatabaseModels(true)
- const video = await VideoModel.loadByUUIDWithFile(program['video'])
+ const video = await VideoModel.loadByUUID(program['video'])
if (!video) throw new Error('Video not found.')
const dataInput: VideoTranscodingPayload = program.resolution !== undefined
import { CONFIG } from '../server/initializers/config'
import { VideoModel } from '../server/models/video/video'
import { initDatabaseModels } from '../server/initializers'
-import { remove, readdir } from 'fs-extra'
+import { readdir, remove } from 'fs-extra'
import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
+import * as Bluebird from 'bluebird'
import { getUUIDFromFilename } from '../server/helpers/utils'
+import { ThumbnailModel } from '../server/models/video/thumbnail'
+import { AvatarModel } from '../server/models/avatar/avatar'
run()
.then(() => process.exit(0))
async function run () {
await initDatabaseModels(true)
- const storageOnlyOwnedToPrune = [
- CONFIG.STORAGE.VIDEOS_DIR,
- CONFIG.STORAGE.TORRENTS_DIR,
- CONFIG.STORAGE.REDUNDANCY_DIR
- ]
+ let toDelete: string[] = []
- const storageForAllToPrune = [
- CONFIG.STORAGE.PREVIEWS_DIR,
- CONFIG.STORAGE.THUMBNAILS_DIR
- ]
+ toDelete = toDelete.concat(
+ await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
+ await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
- let toDelete: string[] = []
- for (const directory of storageOnlyOwnedToPrune) {
- toDelete = toDelete.concat(await pruneDirectory(directory, true))
- }
+ await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
- for (const directory of storageForAllToPrune) {
- toDelete = toDelete.concat(await pruneDirectory(directory, false))
- }
+ await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
+ await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
+
+ await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
+ )
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
}
}
-async function pruneDirectory (directory: string, onlyOwned = false) {
+type ExistFun = (file: string) => Promise<boolean>
+async function pruneDirectory (directory: string, existFun: ExistFun) {
const files = await readdir(directory)
const toDelete: string[] = []
- for (const file of files) {
+ await Bluebird.map(files, async file => {
+ if (await existFun(file) !== true) {
+ toDelete.push(join(directory, file))
+ }
+ }, { concurrency: 20 })
+
+ return toDelete
+}
+
+function doesVideoExist (keepOnlyOwned: boolean) {
+ return async (file: string) => {
const uuid = getUUIDFromFilename(file)
- let video: VideoModel
- let localRedundancy: boolean
+ const video = await VideoModel.loadByUUID(uuid)
- if (uuid) {
- video = await VideoModel.loadByUUIDWithFile(uuid)
- localRedundancy = await VideoRedundancyModel.isLocalByVideoUUIDExists(uuid)
- }
+ return video && (keepOnlyOwned === false || video.isOwned())
+ }
+}
- if (
- !uuid ||
- !video ||
- (onlyOwned === true && (video.isOwned() === false && localRedundancy === false))
- ) {
- toDelete.push(join(directory, file))
+function doesThumbnailExist (keepOnlyOwned: boolean) {
+ return async (file: string) => {
+ const thumbnail = await ThumbnailModel.loadByName(file)
+ if (!thumbnail) return false
+
+ if (keepOnlyOwned) {
+ const video = await VideoModel.load(thumbnail.videoId)
+ if (video.isOwned() === false) return false
}
+
+ return true
}
+}
- return toDelete
+async function doesAvatarExist (file: string) {
+ const avatar = await AvatarModel.loadByName(file)
+
+ return !!avatar
+}
+
+async function doesRedundancyExist (file: string) {
+ const uuid = getUUIDFromFilename(file)
+ const video = await VideoModel.loadWithFiles(uuid)
+
+ if (!video) return false
+
+ const isPlaylist = file.includes('.') === false
+
+ if (isPlaylist) {
+ const p = video.getHLSPlaylist()
+ if (!p) return false
+
+ const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
+ return !!redundancy
+ }
+
+ const resolution = parseInt(file.split('-')[5], 10)
+ if (isNaN(resolution)) {
+ console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
+ return true
+ }
+
+ const videoFile = video.getFile(resolution)
+ if (!videoFile) {
+ console.error('Cannot find file of video %s - %d', video.url, resolution)
+ return true
+ }
+
+ const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
+ return !!redundancy
}
async function askConfirmation () {
logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl)
- await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
+ try {
+ await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
+ } catch (err) {
+ logger.warn('Cannot process remote avatar %s.', avatar.fileUrl, { err })
+ return res.sendStatus(404)
+ }
avatar.onDisk = true
avatar.save()
}
async getFilePathImpl (videoUUID: string) {
- const video = await VideoModel.loadByUUIDWithFile(videoUUID)
+ const video = await VideoModel.loadByUUID(videoUUID)
if (!video) return undefined
if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() }
.catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err))
}
+ static loadByName (filename: string) {
+ const query = {
+ where: {
+ filename
+ }
+ }
+
+ return ThumbnailModel.findOne(query)
+ }
+
static generateDefaultPreviewName (videoUUID: string) {
return videoUUID + '.jpg'
}
import { ThumbnailModel } from './thumbnail'
import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
import { createTorrentPromise } from '../../helpers/webtorrent'
+import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [
return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(options)
}
- static loadWithFiles (id: number, t?: Transaction, logging?: boolean) {
+ static loadWithFiles (id: number | string, t?: Transaction, logging?: boolean) {
+ const where = buildWhereIdOrUUID(id)
+
+ const query = {
+ where,
+ transaction: t,
+ logging
+ }
+
return VideoModel.scope([
ScopeNames.WITH_FILES,
ScopeNames.WITH_STREAMING_PLAYLISTS,
ScopeNames.WITH_THUMBNAILS
- ]).findByPk(id, { transaction: t, logging })
+ ]).findOne(query)
}
- static loadByUUIDWithFile (uuid: string) {
+ static loadByUUID (uuid: string) {
const options = {
where: {
uuid
return maxBy(this.VideoFiles, file => file.resolution)
}
- getFile (resolution: VideoResolution) {
+ getFile (resolution: number) {
if (Array.isArray(this.VideoFiles) === false) return undefined
return this.VideoFiles.find(f => f.resolution === resolution)
return `/api/${API_VERSION}/videos/${this.uuid}/description`
}
+ getHLSPlaylist () {
+ if (!this.VideoStreamingPlaylists) return undefined
+
+ return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
+ }
+
removeFile (videoFile: VideoFileModel, isRedundancy = false) {
const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
import './optimize-old-videos'
import './peertube'
import './plugins'
+import './prune-storage'
import './reset-password'
import './update-host'
--- /dev/null
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+import * as chai from 'chai'
+import { waitJobs } from '../../../shared/extra-utils/server/jobs'
+import {
+ buildServerDirectory,
+ cleanupTests,
+ createVideoPlaylist,
+ doubleFollow,
+ execCLI,
+ flushAndRunMultipleServers,
+ getAccount,
+ getEnvCli,
+ ServerInfo,
+ setAccessTokensToServers, setDefaultVideoChannel,
+ updateMyAvatar,
+ uploadVideo,
+ wait
+} from '../../../shared/extra-utils'
+import { Account, VideoPlaylistPrivacy } from '../../../shared/models'
+import { createFile, readdir } from 'fs-extra'
+import * as uuidv4 from 'uuid/v4'
+import { join } from 'path'
+import * as request from 'supertest'
+
+const expect = chai.expect
+
+async function countFiles (internalServerNumber: number, directory: string) {
+ const files = await readdir(buildServerDirectory(internalServerNumber, directory))
+
+ return files.length
+}
+
+async function assertNotExists (internalServerNumber: number, directory: string, substring: string) {
+ const files = await readdir(buildServerDirectory(internalServerNumber, directory))
+
+ for (const f of files) {
+ expect(f).to.not.contain(substring)
+ }
+}
+
+async function assertCountAreOkay (servers: ServerInfo[]) {
+ for (const server of servers) {
+ const videosCount = await countFiles(server.internalServerNumber, 'videos')
+ expect(videosCount).to.equal(8)
+
+ const torrentsCount = await countFiles(server.internalServerNumber, 'torrents')
+ expect(torrentsCount).to.equal(8)
+
+ const previewsCount = await countFiles(server.internalServerNumber, 'previews')
+ expect(previewsCount).to.equal(2)
+
+ const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails')
+ expect(thumbnailsCount).to.equal(6)
+
+ const avatarsCount = await countFiles(server.internalServerNumber, 'avatars')
+ expect(avatarsCount).to.equal(2)
+ }
+}
+
+describe('Test prune storage scripts', function () {
+ let servers: ServerInfo[]
+ const badNames: { [ directory: string ]: string[] } = {}
+
+ before(async function () {
+ this.timeout(120000)
+
+ servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
+ await setAccessTokensToServers(servers)
+ await setDefaultVideoChannel(servers)
+
+ for (const server of servers) {
+ await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
+ await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
+
+ await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' })
+
+ await createVideoPlaylist({
+ url: server.url,
+ token: server.accessToken,
+ playlistAttrs: {
+ displayName: 'playlist',
+ privacy: VideoPlaylistPrivacy.PUBLIC,
+ videoChannelId: server.videoChannel.id,
+ thumbnailfile: 'thumbnail.jpg'
+ }
+ })
+ }
+
+ await doubleFollow(servers[0], servers[1])
+
+ // Lazy load the remote avatar
+ {
+ const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port)
+ const account: Account = res.body
+ await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200)
+ }
+
+ {
+ const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port)
+ const account: Account = res.body
+ await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200)
+ }
+
+ await wait(1000)
+
+ await waitJobs(servers)
+ })
+
+ it('Should have the files on the disk', async function () {
+ await assertCountAreOkay(servers)
+ })
+
+ it('Should create some dirty files', async function () {
+ for (let i = 0; i < 2; i++) {
+ {
+ const base = buildServerDirectory(servers[0].internalServerNumber, 'videos')
+
+ const n1 = uuidv4() + '.mp4'
+ const n2 = uuidv4() + '.webm'
+
+ await createFile(join(base, n1))
+ await createFile(join(base, n2))
+
+ badNames['videos'] = [ n1, n2 ]
+ }
+
+ {
+ const base = buildServerDirectory(servers[0].internalServerNumber, 'torrents')
+
+ const n1 = uuidv4() + '-240.torrent'
+ const n2 = uuidv4() + '-480.torrent'
+
+ await createFile(join(base, n1))
+ await createFile(join(base, n2))
+
+ badNames['torrents'] = [ n1, n2 ]
+ }
+
+ {
+ const base = buildServerDirectory(servers[0].internalServerNumber, 'thumbnails')
+
+ const n1 = uuidv4() + '.jpg'
+ const n2 = uuidv4() + '.jpg'
+
+ await createFile(join(base, n1))
+ await createFile(join(base, n2))
+
+ badNames['thumbnails'] = [ n1, n2 ]
+ }
+
+ {
+ const base = buildServerDirectory(servers[0].internalServerNumber, 'previews')
+
+ const n1 = uuidv4() + '.jpg'
+ const n2 = uuidv4() + '.jpg'
+
+ await createFile(join(base, n1))
+ await createFile(join(base, n2))
+
+ badNames['previews'] = [ n1, n2 ]
+ }
+
+ {
+ const base = buildServerDirectory(servers[0].internalServerNumber, 'avatars')
+
+ const n1 = uuidv4() + '.png'
+ const n2 = uuidv4() + '.jpg'
+
+ await createFile(join(base, n1))
+ await createFile(join(base, n2))
+
+ badNames['avatars'] = [ n1, n2 ]
+ }
+ }
+ })
+
+ it('Should run prune storage', async function () {
+ this.timeout(30000)
+
+ const env = getEnvCli(servers[0])
+ await execCLI(`echo y | ${env} npm run prune-storage`)
+ })
+
+ it('Should have removed files', async function () {
+ await assertCountAreOkay(servers)
+
+ for (const directory of Object.keys(badNames)) {
+ for (const name of badNames[directory]) {
+ await assertNotExists(servers[0].internalServerNumber, directory, name)
+ }
+ }
+ })
+
+ after(async function () {
+ await cleanupTests(servers)
+ })
+})
return root
}
+function buildServerDirectory (internalServerNumber: number, directory: string) {
+ return join(root(), 'test' + internalServerNumber, directory)
+}
+
async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
const res = await request(url)
.get(imagePath)
export {
dateIsValid,
wait,
+ buildServerDirectory,
webtorrentAdd,
immutableAssign,
testImage,
import * as validator from 'validator'
import { VideoDetails, VideoPrivacy } from '../../models/videos'
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
-import { dateIsValid, webtorrentAdd } from '../miscs/miscs'
+import { dateIsValid, webtorrentAdd, buildServerDirectory } from '../miscs/miscs'
loadLanguages()
join('redundancy', 'hls')
]
) {
- const testDirectory = 'test' + serverNumber
-
for (const directory of directories) {
- const directoryPath = join(root(), testDirectory, directory)
+ const directoryPath = buildServerDirectory(serverNumber, directory)
const directoryExists = await pathExists(directoryPath)
if (directoryExists === false) continue
projects / oweals / peertube.git / commitdiff