diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index e767492bc1a..273e52fe684 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -1,7 +1,7 @@ import * as express from 'express' import { move } from 'fs-extra' import { getLowercaseExtension } from '@server/helpers/core-utils' -import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' +import { getResumableUploadPath, scheduleDeleteResumableUploadMetaFile } from '@server/helpers/upload' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' @@ -35,6 +35,7 @@ import { import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' import { VideoModel } from '../../../models/video/video' import { VideoFileModel } from '../../../models/video/video-file' +import { Redis } from '@server/lib/redis' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') @@ -112,18 +113,21 @@ export async function addVideoLegacy (req: express.Request, res: express.Respons const videoInfo: VideoCreate = req.body const files = req.files - return addVideo({ res, videoPhysicalFile, videoInfo, files }) + const response = addVideo({ res, videoPhysicalFile, videoInfo, files }) + return res.json(response) } -export async function addVideoResumable (_req: express.Request, res: express.Response) { +export async function addVideoResumable (req: express.Request, res: express.Response) { const videoPhysicalFile = res.locals.videoFileResumable const videoInfo = videoPhysicalFile.metadata const files = { previewfile: videoInfo.previewfile } // Don't need the meta file anymore - await deleteResumableUploadMetaFile(videoPhysicalFile.path) + scheduleDeleteResumableUploadMetaFile(videoPhysicalFile.path) - return addVideo({ res, videoPhysicalFile, videoInfo, files }) + const response = await addVideo({ res, videoPhysicalFile, videoInfo, files }) + await Redis.Instance.setUploadSession(req.query.upload_id, response.video) + return res.json(response) } async function addVideo (options: { @@ -215,12 +219,12 @@ async function addVideo (options: { Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) - return res.json({ + return { video: { id: videoCreated.id, uuid: videoCreated.uuid } - }) + } } async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts index 030a6b7d547..71d90f991e0 100644 --- a/server/helpers/upload.ts +++ b/server/helpers/upload.ts @@ -1,3 +1,4 @@ +import { JobQueue } from '@server/lib/job-queue' import { METAFILE_EXTNAME } from '@uploadx/core' import { remove } from 'fs-extra' import { join } from 'path' @@ -13,9 +14,15 @@ function deleteResumableUploadMetaFile (filepath: string) { return remove(filepath + METAFILE_EXTNAME) } +function scheduleDeleteResumableUploadMetaFile (filepath: string) { + const payload = { filepath } + JobQueue.Instance.createJob({ type: 'delete-resumable-upload-meta-file', payload }, { delay: 900 * 1000 }) // executed in 15 min +} + // --------------------------------------------------------------------------- export { getResumableUploadPath, - deleteResumableUploadMetaFile + deleteResumableUploadMetaFile, + scheduleDeleteResumableUploadMetaFile } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index d41d0e0565f..cea350edb0f 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -146,7 +146,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { 'videos-views': 1, 'activitypub-refresher': 1, 'video-redundancy': 1, - 'video-live-ending': 1 + 'video-live-ending': 1, + 'delete-resumable-upload-meta-file': 1 } // Excluded keys are jobs that can be configured by admins const JOB_CONCURRENCY: { [id in Exclude]: number } = { @@ -161,7 +162,8 @@ const JOB_CONCURRENCY: { [id in Exclude Promise } = { 'activitypub-refresher': refreshAPObject, 'video-live-ending': processVideoLiveEnding, 'actor-keys': processActorKeys, - 'video-redundancy': processVideoRedundancy + 'video-redundancy': processVideoRedundancy, + 'delete-resumable-upload-meta-file': processDeleteResumableUploadMetaFile } const jobTypes: JobType[] = [ @@ -87,7 +91,8 @@ const jobTypes: JobType[] = [ 'activitypub-refresher', 'video-redundancy', 'actor-keys', - 'video-live-ending' + 'video-live-ending', + 'delete-resumable-upload-meta-file' ] class JobQueue { diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 62641e313af..a3a8727922b 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -9,7 +9,8 @@ import { USER_PASSWORD_CREATE_LIFETIME, VIEW_LIFETIME, WEBSERVER, - TRACKER_RATE_LIMITS + TRACKER_RATE_LIMITS, + RESUMABLE_UPLOAD_SESSION_LIFETIME } from '../initializers/constants' import { CONFIG } from '../initializers/config' @@ -202,6 +203,29 @@ class Redis { ]) } + /* ************ Resumable uploads final responses ************ */ + + setUploadSession (uploadId: string, video?: { id: number, uuid: string }) { + return this.setObject( + uploadId, + video + ? { + id: video.id.toString(), + uuid: video.uuid + } + : null, + RESUMABLE_UPLOAD_SESSION_LIFETIME + ) + } + + doesUploadSessionExist (uploadId: string) { + return this.exists(uploadId) + } + + getUploadSession (uploadId: string) { + return this.getValue(uploadId) + } + /* ************ Keys generation ************ */ generateCachedRouteKey (req: express.Request) { diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index 7f278c9f65f..7d440e2101b 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -1,5 +1,6 @@ import * as express from 'express' import { body, header, param, query, ValidationChain } from 'express-validator' +import { Redis } from '@server/lib/redis' import { getResumableUploadPath } from '@server/helpers/upload' import { isAbleToUploadVideo } from '@server/lib/user' import { getServerActor } from '@server/models/application/application' @@ -109,12 +110,28 @@ const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ const videosAddResumableValidator = [ async (req: express.Request, res: express.Response, next: express.NextFunction) => { const user = res.locals.oauth.token.User - const body: express.CustomUploadXFile = req.body const file = { ...body, duration: undefined, path: getResumableUploadPath(body.id), filename: body.metadata.filename } - const cleanup = () => deleteFileAndCatch(file.path) + const uploadId = req.query.upload_id + const sessionExists = await Redis.Instance.doesUploadSessionExist(uploadId) + if (sessionExists) { + const sessionResponse = await Redis.Instance.getUploadSession(uploadId) + if (!sessionResponse) { + res.setHeader('Retry-After', 300) // ask to retry after 5 min, knowing the upload_id is kept for up to 15 min after completion + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'The upload is already being processed' + }) + } else { + res.json({ video: sessionResponse }) + } + return + } else { + await Redis.Instance.setUploadSession(uploadId) + } + if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup() try { diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index e4acfee8d0a..11dd7524cfc 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -19,6 +19,7 @@ export type JobType = | 'video-redundancy' | 'video-live-ending' | 'actor-keys' + | 'delete-resumable-upload-meta-file' export interface Job { id: number @@ -137,3 +138,7 @@ export interface VideoLiveEndingPayload { export interface ActorKeysPayload { actorId: number } + +export interface DeleteResumableUploadMetaFilePayload { + filepath: string +} diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index b43b6bfa0fd..f43943ec139 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -2039,7 +2039,7 @@ paths: '404': description: upload not found '409': - description: chunk doesn't match range + description: chunk doesn't match range or upload being processed '422': description: video unreadable '429':