はじまりの大地
このコミットが含まれているのは:
@@ -0,0 +1,202 @@
|
||||
import Bluebird from 'bluebird'
|
||||
import { Job } from 'bullmq'
|
||||
import {
|
||||
isAnnounceActivityValid,
|
||||
isDislikeActivityValid,
|
||||
isLikeActivityValid
|
||||
} from '@server/helpers/custom-validators/activitypub/activity.js'
|
||||
import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments.js'
|
||||
import { PeerTubeRequestError } from '@server/helpers/requests.js'
|
||||
import { AP_CLEANER } from '@server/initializers/constants.js'
|
||||
import { fetchAP } from '@server/lib/activitypub/activity.js'
|
||||
import { checkUrlsSameHost } from '@server/lib/activitypub/url.js'
|
||||
import { Redis } from '@server/lib/redis.js'
|
||||
import { VideoCommentModel } from '@server/models/video/video-comment.js'
|
||||
import { VideoShareModel } from '@server/models/video/video-share.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { HttpStatusCode } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js'
|
||||
|
||||
const lTags = loggerTagsFactory('ap-cleaner')
|
||||
|
||||
// Job to clean remote interactions off local videos
|
||||
|
||||
async function processActivityPubCleaner (_job: Job) {
|
||||
logger.info('Processing ActivityPub cleaner.', lTags())
|
||||
|
||||
{
|
||||
const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos()
|
||||
const { bodyValidator, deleter, updater } = rateOptionsFactory()
|
||||
|
||||
await Bluebird.map(rateUrls, async rateUrl => {
|
||||
// TODO: remove when https://github.com/mastodon/mastodon/issues/13571 is fixed
|
||||
if (rateUrl.includes('#')) return
|
||||
|
||||
const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter })
|
||||
|
||||
if (result?.status === 'deleted') {
|
||||
const { videoId, type } = result.data
|
||||
|
||||
await VideoModel.syncLocalRates(videoId, type, undefined)
|
||||
}
|
||||
}, { concurrency: AP_CLEANER.CONCURRENCY })
|
||||
}
|
||||
|
||||
{
|
||||
const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos()
|
||||
const { bodyValidator, deleter, updater } = shareOptionsFactory()
|
||||
|
||||
await Bluebird.map(shareUrls, async shareUrl => {
|
||||
await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter })
|
||||
}, { concurrency: AP_CLEANER.CONCURRENCY })
|
||||
}
|
||||
|
||||
{
|
||||
const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos()
|
||||
const { bodyValidator, deleter, updater } = commentOptionsFactory()
|
||||
|
||||
await Bluebird.map(commentUrls, async commentUrl => {
|
||||
await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter })
|
||||
}, { concurrency: AP_CLEANER.CONCURRENCY })
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubCleaner
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function updateObjectIfNeeded <T> (options: {
|
||||
url: string
|
||||
bodyValidator: (body: any) => boolean
|
||||
updater: (url: string, newUrl: string) => Promise<T>
|
||||
deleter: (url: string) => Promise<T> }
|
||||
): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
|
||||
const { url, bodyValidator, updater, deleter } = options
|
||||
|
||||
const on404OrTombstone = async () => {
|
||||
logger.info('Removing remote AP object %s.', url, lTags(url))
|
||||
const data = await deleter(url)
|
||||
|
||||
return { status: 'deleted' as 'deleted', data }
|
||||
}
|
||||
|
||||
try {
|
||||
const { body } = await fetchAP<any>(url)
|
||||
|
||||
// If not same id, check same host and update
|
||||
if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
|
||||
|
||||
if (body.type === 'Tombstone') {
|
||||
return on404OrTombstone()
|
||||
}
|
||||
|
||||
const newUrl = body.id
|
||||
if (newUrl !== url) {
|
||||
if (checkUrlsSameHost(newUrl, url) !== true) {
|
||||
throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
|
||||
}
|
||||
|
||||
logger.info('Updating remote AP object %s.', url, lTags(url))
|
||||
const data = await updater(url, newUrl)
|
||||
|
||||
return { status: 'updated', data }
|
||||
}
|
||||
|
||||
return null
|
||||
} catch (err) {
|
||||
// Does not exist anymore, remove entry
|
||||
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
|
||||
return on404OrTombstone()
|
||||
}
|
||||
|
||||
logger.debug('Remote AP object %s is unavailable.', url, lTags(url))
|
||||
|
||||
const unavailability = await Redis.Instance.addAPUnavailability(url)
|
||||
if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) {
|
||||
logger.info('Removing unavailable AP resource %s.', url, lTags(url))
|
||||
return on404OrTombstone()
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function rateOptionsFactory () {
|
||||
return {
|
||||
bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body),
|
||||
|
||||
updater: async (url: string, newUrl: string) => {
|
||||
const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
|
||||
rate.url = newUrl
|
||||
|
||||
const videoId = rate.videoId
|
||||
const type = rate.type
|
||||
|
||||
await rate.save()
|
||||
|
||||
return { videoId, type }
|
||||
},
|
||||
|
||||
deleter: async (url) => {
|
||||
const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
|
||||
|
||||
const videoId = rate.videoId
|
||||
const type = rate.type
|
||||
|
||||
await rate.destroy()
|
||||
|
||||
return { videoId, type }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function shareOptionsFactory () {
|
||||
return {
|
||||
bodyValidator: (body: any) => isAnnounceActivityValid(body),
|
||||
|
||||
updater: async (url: string, newUrl: string) => {
|
||||
const share = await VideoShareModel.loadByUrl(url, undefined)
|
||||
share.url = newUrl
|
||||
|
||||
await share.save()
|
||||
|
||||
return undefined
|
||||
},
|
||||
|
||||
deleter: async (url) => {
|
||||
const share = await VideoShareModel.loadByUrl(url, undefined)
|
||||
|
||||
await share.destroy()
|
||||
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function commentOptionsFactory () {
|
||||
return {
|
||||
bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body),
|
||||
|
||||
updater: async (url: string, newUrl: string) => {
|
||||
const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(url)
|
||||
comment.url = newUrl
|
||||
|
||||
await comment.save()
|
||||
|
||||
return undefined
|
||||
},
|
||||
|
||||
deleter: async (url) => {
|
||||
const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(url)
|
||||
|
||||
await comment.destroy()
|
||||
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js'
|
||||
import { ActivitypubFollowPayload } from '@peertube/peertube-models'
|
||||
import { sanitizeHost } from '../../../helpers/core-utils.js'
|
||||
import { retryTransactionWrapper } from '../../../helpers/database-utils.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { REMOTE_SCHEME, SERVER_ACTOR_NAME, WEBSERVER } from '../../../initializers/constants.js'
|
||||
import { sequelizeTypescript } from '../../../initializers/database.js'
|
||||
import { ActorModel } from '../../../models/actor/actor.js'
|
||||
import { ActorFollowModel } from '../../../models/actor/actor-follow.js'
|
||||
import { MActor, MActorFull } from '../../../types/models/index.js'
|
||||
import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors/index.js'
|
||||
import { sendFollow } from '../../activitypub/send/index.js'
|
||||
import { Notifier } from '../../notifier/index.js'
|
||||
import { getApplicationActorOfHost } from '@server/helpers/activity-pub-utils.js'
|
||||
|
||||
async function processActivityPubFollow (job: Job) {
|
||||
const payload = job.data as ActivitypubFollowPayload
|
||||
const host = payload.host
|
||||
|
||||
const handle = host
|
||||
? `${payload.name}@${host}`
|
||||
: payload.name
|
||||
|
||||
logger.info('Processing ActivityPub follow in job %s.', job.id)
|
||||
|
||||
let targetActor: MActorFull
|
||||
if (!host || host === WEBSERVER.HOST) {
|
||||
if (!payload.name) throw new Error('Payload name is mandatory for local follow')
|
||||
|
||||
targetActor = await ActorModel.loadLocalByName(payload.name)
|
||||
} else {
|
||||
const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP)
|
||||
|
||||
let actorUrl: string
|
||||
|
||||
try {
|
||||
if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost)
|
||||
if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost)
|
||||
|
||||
targetActor = await getOrCreateAPActor(actorUrl, 'all')
|
||||
} catch (err) {
|
||||
logger.warn(`Do not follow ${handle} because we could not find the actor URL (in database or using webfinger)`)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (!targetActor) {
|
||||
logger.warn(`Do not follow ${handle} because we could not fetch/load the actor`)
|
||||
return
|
||||
}
|
||||
|
||||
if (payload.assertIsChannel && !targetActor.VideoChannel) {
|
||||
logger.warn(`Do not follow ${handle} because it is not a channel.`)
|
||||
return
|
||||
}
|
||||
|
||||
const fromActor = await ActorModel.load(payload.followerActorId)
|
||||
|
||||
return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow)
|
||||
}
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubFollow
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function follow (fromActor: MActor, targetActor: MActorFull, isAutoFollow = false) {
|
||||
if (fromActor.id === targetActor.id) {
|
||||
throw new Error('Follower is the same as target actor.')
|
||||
}
|
||||
|
||||
// Same server, direct accept
|
||||
const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending'
|
||||
|
||||
const actorFollow = await sequelizeTypescript.transaction(async t => {
|
||||
const [ actorFollow ] = await ActorFollowModel.findOrCreateCustom({
|
||||
byActor: fromActor,
|
||||
state,
|
||||
targetActor,
|
||||
activityId: getLocalActorFollowActivityPubUrl(fromActor, targetActor),
|
||||
transaction: t
|
||||
})
|
||||
|
||||
// Send a notification to remote server if our follow is not already accepted
|
||||
if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t)
|
||||
|
||||
return actorFollow
|
||||
})
|
||||
|
||||
const followerFull = await ActorModel.loadFull(fromActor.id)
|
||||
|
||||
const actorFollowFull = Object.assign(actorFollow, {
|
||||
ActorFollowing: targetActor,
|
||||
ActorFollower: followerFull
|
||||
})
|
||||
|
||||
if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollowFull)
|
||||
if (isAutoFollow === true) Notifier.Instance.notifyOfAutoInstanceFollowing(actorFollowFull)
|
||||
|
||||
return actorFollow
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { ActivitypubHttpBroadcastPayload } from '@peertube/peertube-models'
|
||||
import { buildGlobalHTTPHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/http.js'
|
||||
import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache.js'
|
||||
import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
// Prefer using a worker thread for HTTP requests because on high load we may have to sign many requests, which can be CPU intensive
|
||||
|
||||
async function processActivityPubHttpSequentialBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) {
|
||||
logger.info('Processing ActivityPub broadcast in job %s.', job.id)
|
||||
|
||||
const requestOptions = await buildRequestOptions(job.data)
|
||||
|
||||
const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
|
||||
|
||||
return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls)
|
||||
}
|
||||
|
||||
async function processActivityPubParallelHttpBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) {
|
||||
logger.info('Processing ActivityPub parallel broadcast in job %s.', job.id)
|
||||
|
||||
const requestOptions = await buildRequestOptions(job.data)
|
||||
|
||||
const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
|
||||
|
||||
return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubHttpSequentialBroadcast,
|
||||
processActivityPubParallelHttpBroadcast
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) {
|
||||
const body = await computeBody(payload)
|
||||
const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true })
|
||||
|
||||
return {
|
||||
method: 'POST' as 'POST',
|
||||
json: body,
|
||||
httpSignature: httpSignatureOptions,
|
||||
headers: await buildGlobalHTTPHeaders(body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { ActivitypubHttpFetcherPayload, FetchType } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
import { VideoCommentModel } from '../../../models/video/video-comment.js'
|
||||
import { VideoShareModel } from '../../../models/video/video-share.js'
|
||||
import { MVideoFullLight } from '../../../types/models/index.js'
|
||||
import { crawlCollectionPage } from '../../activitypub/crawl.js'
|
||||
import { createAccountPlaylists } from '../../activitypub/playlists/index.js'
|
||||
import { processActivities } from '../../activitypub/process/index.js'
|
||||
import { addVideoShares } from '../../activitypub/share.js'
|
||||
import { addVideoComments } from '../../activitypub/video-comments.js'
|
||||
|
||||
async function processActivityPubHttpFetcher (job: Job) {
|
||||
logger.info('Processing ActivityPub fetcher in job %s.', job.id)
|
||||
|
||||
const payload = job.data as ActivitypubHttpFetcherPayload
|
||||
|
||||
let video: MVideoFullLight
|
||||
if (payload.videoId) video = await VideoModel.loadFull(payload.videoId)
|
||||
|
||||
const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
|
||||
'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }),
|
||||
'video-shares': items => addVideoShares(items, video),
|
||||
'video-comments': items => addVideoComments(items),
|
||||
'account-playlists': items => createAccountPlaylists(items)
|
||||
}
|
||||
|
||||
const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = {
|
||||
'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate),
|
||||
'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
|
||||
}
|
||||
|
||||
return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubHttpFetcher
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { ActivitypubHttpUnicastPayload } from '@peertube/peertube-models'
|
||||
import { buildGlobalHTTPHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/http.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { ActorFollowHealthCache } from '../../actor-follow-health-cache.js'
|
||||
import { httpUnicastFromWorker } from '@server/lib/worker/parent-process.js'
|
||||
|
||||
async function processActivityPubHttpUnicast (job: Job) {
|
||||
logger.info('Processing ActivityPub unicast in job %s.', job.id)
|
||||
|
||||
const payload = job.data as ActivitypubHttpUnicastPayload
|
||||
const uri = payload.uri
|
||||
|
||||
const body = await computeBody(payload)
|
||||
const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true })
|
||||
|
||||
const options = {
|
||||
method: 'POST' as 'POST',
|
||||
json: body,
|
||||
httpSignature: httpSignatureOptions,
|
||||
headers: await buildGlobalHTTPHeaders(body)
|
||||
}
|
||||
|
||||
try {
|
||||
await httpUnicastFromWorker({ uri, requestOptions: options })
|
||||
ActorFollowHealthCache.Instance.updateActorFollowsHealth([ uri ], [])
|
||||
} catch (err) {
|
||||
ActorFollowHealthCache.Instance.updateActorFollowsHealth([], [ uri ])
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubHttpUnicast
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { RefreshPayload } from '@peertube/peertube-models'
|
||||
import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists/index.js'
|
||||
import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders/index.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { ActorModel } from '../../../models/actor/actor.js'
|
||||
import { VideoPlaylistModel } from '../../../models/video/video-playlist.js'
|
||||
import { refreshActorIfNeeded } from '../../activitypub/actors/index.js'
|
||||
|
||||
async function refreshAPObject (job: Job) {
|
||||
const payload = job.data as RefreshPayload
|
||||
|
||||
logger.info('Processing AP refresher in job %s for %s.', job.id, payload.url)
|
||||
|
||||
if (payload.type === 'video') return refreshVideo(payload.url)
|
||||
if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
|
||||
if (payload.type === 'actor') return refreshActor(payload.url)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
refreshAPObject
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function refreshVideo (videoUrl: string) {
|
||||
const fetchType = 'all'
|
||||
const syncParam = { rates: true, shares: true, comments: true }
|
||||
|
||||
const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
|
||||
if (videoFromDatabase) {
|
||||
const refreshOptions = {
|
||||
video: videoFromDatabase,
|
||||
fetchedType: fetchType as VideoLoadByUrlType,
|
||||
syncParam
|
||||
}
|
||||
|
||||
await refreshVideoIfNeeded(refreshOptions)
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshActor (actorUrl: string) {
|
||||
const fetchType = 'all'
|
||||
const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl)
|
||||
|
||||
if (actor) {
|
||||
await refreshActorIfNeeded({ actor, fetchedType: fetchType })
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshVideoPlaylist (playlistUrl: string) {
|
||||
const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
|
||||
|
||||
if (playlist) {
|
||||
await refreshVideoPlaylistIfNeeded(playlist)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors/index.js'
|
||||
import { ActorModel } from '@server/models/actor/actor.js'
|
||||
import { ActorKeysPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processActorKeys (job: Job) {
|
||||
const payload = job.data as ActorKeysPayload
|
||||
logger.info('Processing actor keys in job %s.', job.id)
|
||||
|
||||
const actor = await ActorModel.load(payload.actorId)
|
||||
|
||||
await generateAndSaveActorKeys(actor)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActorKeys
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { logger } from '@server/helpers/logger.js'
|
||||
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
|
||||
import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@peertube/peertube-models'
|
||||
|
||||
export async function processAfterVideoChannelImport (job: Job) {
|
||||
const payload = job.data as AfterVideoChannelImportPayload
|
||||
if (!payload.channelSyncId) return
|
||||
|
||||
logger.info('Processing after video channel import in job %s.', job.id)
|
||||
|
||||
const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
|
||||
if (!sync) {
|
||||
logger.error('Unknown sync id %d.', payload.channelSyncId)
|
||||
return
|
||||
}
|
||||
|
||||
const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
|
||||
|
||||
let errors = 0
|
||||
let successes = 0
|
||||
|
||||
for (const value of Object.values(childrenValues)) {
|
||||
if (value.resultType === 'success') successes++
|
||||
else if (value.resultType === 'error') errors++
|
||||
}
|
||||
|
||||
if (errors > 0) {
|
||||
sync.state = VideoChannelSyncState.FAILED
|
||||
logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
|
||||
} else {
|
||||
sync.state = VideoChannelSyncState.SYNCED
|
||||
logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
|
||||
}
|
||||
|
||||
await sync.save()
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { CreateUserExportPayload } from '@peertube/peertube-models'
|
||||
import { UserExportModel } from '@server/models/user/user-export.js'
|
||||
import { UserExporter } from '@server/lib/user-import-export/user-exporter.js'
|
||||
import { Emailer } from '@server/lib/emailer.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-export')
|
||||
|
||||
export async function processCreateUserExport (job: Job): Promise<void> {
|
||||
const payload = job.data as CreateUserExportPayload
|
||||
const exportModel = await UserExportModel.load(payload.userExportId)
|
||||
|
||||
logger.info('Processing create user export %s in job %s.', payload.userExportId, job.id, lTags())
|
||||
|
||||
if (!exportModel) {
|
||||
logger.info(`User export ${payload.userExportId} does not exist anymore, do not create user export.`, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
const exporter = new UserExporter()
|
||||
|
||||
try {
|
||||
await exporter.export(exportModel)
|
||||
|
||||
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
|
||||
|
||||
logger.info(`User export ${payload.userExportId} has been created`, lTags())
|
||||
} catch (err) {
|
||||
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { EmailPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { Emailer } from '../../emailer.js'
|
||||
|
||||
async function processEmail (job: Job) {
|
||||
const payload = job.data as EmailPayload
|
||||
logger.info('Processing email in job %s.', job.id)
|
||||
|
||||
return Emailer.Instance.sendMail(payload)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processEmail
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
|
||||
import { sequelizeTypescript } from '@server/initializers/database.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { FederateVideoPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
function processFederateVideo (job: Job) {
|
||||
const payload = job.data as FederateVideoPayload
|
||||
|
||||
logger.info('Processing video federation in job %s.', job.id)
|
||||
|
||||
return retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID, t)
|
||||
if (!video) return
|
||||
|
||||
return federateVideoIfNeeded(video, payload.isNewVideoForFederation, t)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processFederateVideo
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo, isAudioFile } from '@peertube/peertube-ffmpeg'
|
||||
import { GenerateStoryboardPayload } from '@peertube/peertube-models'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
|
||||
import { generateImageFilename } from '@server/helpers/image-utils.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { deleteFileAndCatch } from '@server/helpers/utils.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { STORYBOARD } from '@server/initializers/constants.js'
|
||||
import { sequelizeTypescript } from '@server/initializers/database.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
|
||||
import { StoryboardModel } from '@server/models/video/storyboard.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideo } from '@server/types/models/index.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { join } from 'path'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('storyboard')
|
||||
|
||||
async function processGenerateStoryboard (job: Job): Promise<void> {
|
||||
const payload = job.data as GenerateStoryboardPayload
|
||||
const lTags = lTagsBase(payload.videoUUID)
|
||||
|
||||
logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags)
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
|
||||
|
||||
try {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
if (!video) {
|
||||
logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
|
||||
return
|
||||
}
|
||||
|
||||
const inputFile = video.getMaxQualityFile()
|
||||
|
||||
await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
|
||||
const probe = await ffprobePromise(videoPath)
|
||||
const isAudio = await isAudioFile(videoPath, probe)
|
||||
|
||||
if (isAudio) {
|
||||
logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
|
||||
return
|
||||
}
|
||||
|
||||
const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)
|
||||
let spriteHeight: number
|
||||
let spriteWidth: number
|
||||
|
||||
if (videoStreamInfo.isPortraitMode) {
|
||||
spriteHeight = STORYBOARD.SPRITE_MAX_SIZE
|
||||
spriteWidth = Math.round(spriteHeight * videoStreamInfo.ratio)
|
||||
} else {
|
||||
spriteWidth = STORYBOARD.SPRITE_MAX_SIZE
|
||||
spriteHeight = Math.round(spriteWidth / videoStreamInfo.ratio)
|
||||
}
|
||||
|
||||
const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
|
||||
|
||||
const filename = generateImageFilename()
|
||||
const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
|
||||
|
||||
const { totalSprites, spriteDuration } = buildSpritesMetadata({ video })
|
||||
if (totalSprites === 0) {
|
||||
logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags)
|
||||
return
|
||||
}
|
||||
|
||||
const spritesCount = findGridSize({
|
||||
toFind: totalSprites,
|
||||
maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
|
||||
})
|
||||
|
||||
logger.debug(
|
||||
'Generating storyboard from video of %s to %s', video.uuid, destination,
|
||||
{ ...lTags, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
|
||||
)
|
||||
|
||||
await ffmpeg.generateStoryboardFromVideo({
|
||||
destination,
|
||||
path: videoPath,
|
||||
inputFileMutexReleaser,
|
||||
sprites: {
|
||||
size: {
|
||||
height: spriteHeight,
|
||||
width: spriteWidth
|
||||
},
|
||||
count: spritesCount,
|
||||
duration: spriteDuration
|
||||
}
|
||||
})
|
||||
|
||||
const imageSize = await getImageSizeFromWorker(destination)
|
||||
|
||||
await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async transaction => {
|
||||
const videoStillExists = await VideoModel.load(video.id, transaction)
|
||||
if (!videoStillExists) {
|
||||
logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
|
||||
deleteFileAndCatch(destination)
|
||||
return
|
||||
}
|
||||
|
||||
const existing = await StoryboardModel.loadByVideo(video.id, transaction)
|
||||
if (existing) await existing.destroy({ transaction })
|
||||
|
||||
await StoryboardModel.create({
|
||||
filename,
|
||||
totalHeight: imageSize.height,
|
||||
totalWidth: imageSize.width,
|
||||
spriteHeight,
|
||||
spriteWidth,
|
||||
spriteDuration,
|
||||
videoId: video.id
|
||||
}, { transaction })
|
||||
|
||||
logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags)
|
||||
|
||||
if (payload.federate) {
|
||||
await federateVideoIfNeeded(video, false, transaction)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processGenerateStoryboard
|
||||
}
|
||||
|
||||
function buildSpritesMetadata (options: {
|
||||
video: MVideo
|
||||
}) {
|
||||
const { video } = options
|
||||
|
||||
if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }
|
||||
|
||||
const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)
|
||||
|
||||
const spriteDuration = Math.ceil(video.duration / maxSprites)
|
||||
const totalSprites = Math.ceil(video.duration / spriteDuration)
|
||||
|
||||
// We can generate a single line so we don't need a prime number
|
||||
if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }
|
||||
|
||||
return { spriteDuration, totalSprites: findNearestGridPrime(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) }
|
||||
}
|
||||
|
||||
function findGridSize (options: {
|
||||
toFind: number
|
||||
maxEdgeCount: number
|
||||
}) {
|
||||
const { toFind, maxEdgeCount } = options
|
||||
|
||||
for (let i = 1; i <= maxEdgeCount; i++) {
|
||||
for (let j = i; j <= maxEdgeCount; j++) {
|
||||
if (toFind === i * j) return { width: j, height: i }
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
|
||||
}
|
||||
|
||||
function findNearestGridPrime (value: number, maxMultiplier: number) {
|
||||
for (let i = value; i--; i > 0) {
|
||||
if (!isPrimeWithin(i, maxMultiplier)) return i
|
||||
}
|
||||
|
||||
throw new Error('Could not find prime number below ' + value)
|
||||
}
|
||||
|
||||
function isPrimeWithin (value: number, maxMultiplier: number) {
|
||||
if (value < 2) return false
|
||||
|
||||
for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) {
|
||||
if (value % i === 0 && value / i <= maxMultiplier) return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { ImportUserArchivePayload } from '@peertube/peertube-models'
|
||||
import { UserImportModel } from '@server/models/user/user-import.js'
|
||||
import { UserImporter } from '@server/lib/user-import-export/user-importer.js'
|
||||
import { Emailer } from '@server/lib/emailer.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
export async function processImportUserArchive (job: Job): Promise<void> {
|
||||
const payload = job.data as ImportUserArchivePayload
|
||||
const importModel = await UserImportModel.load(payload.userImportId)
|
||||
|
||||
logger.info(`Processing importing user archive ${payload.userImportId} in job ${job.id}`, lTags())
|
||||
|
||||
if (!importModel) {
|
||||
logger.info(`User import ${payload.userImportId} does not exist anymore, do not create import data.`, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
const exporter = new UserImporter()
|
||||
await exporter.import(importModel)
|
||||
|
||||
try {
|
||||
await Emailer.Instance.addUserImportSuccessJob(importModel)
|
||||
|
||||
logger.info(`User import ${payload.userImportId} ended`, lTags())
|
||||
} catch (err) {
|
||||
await Emailer.Instance.addUserImportErroredJob(importModel)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { extractVideo } from '@server/helpers/video.js'
|
||||
import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||
import { ManageVideoTorrentPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processManageVideoTorrent (job: Job) {
|
||||
const payload = job.data as ManageVideoTorrentPayload
|
||||
logger.info('Processing torrent in job %s.', job.id)
|
||||
|
||||
if (payload.action === 'create') return doCreateAction(payload)
|
||||
if (payload.action === 'update-metadata') return doUpdateMetadataAction(payload)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processManageVideoTorrent
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'create' }) {
|
||||
const [ video, file ] = await Promise.all([
|
||||
loadVideoOrLog(payload.videoId),
|
||||
loadFileOrLog(payload.videoFileId)
|
||||
])
|
||||
|
||||
if (!video || !file) return
|
||||
|
||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
await file.reload()
|
||||
|
||||
await createTorrentAndSetInfoHash(video, file)
|
||||
|
||||
// Refresh videoFile because the createTorrentAndSetInfoHash could be long
|
||||
const refreshedFile = await VideoFileModel.loadWithVideo(file.id)
|
||||
// File does not exist anymore, remove the generated torrent
|
||||
if (!refreshedFile) return file.removeTorrent()
|
||||
|
||||
refreshedFile.infoHash = file.infoHash
|
||||
refreshedFile.torrentFilename = file.torrentFilename
|
||||
|
||||
await refreshedFile.save()
|
||||
} finally {
|
||||
fileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) {
|
||||
const [ video, streamingPlaylist, file ] = await Promise.all([
|
||||
loadVideoOrLog(payload.videoId),
|
||||
loadStreamingPlaylistOrLog(payload.streamingPlaylistId),
|
||||
loadFileOrLog(payload.videoFileId)
|
||||
])
|
||||
|
||||
if ((!video && !streamingPlaylist) || !file) return
|
||||
|
||||
const extractedVideo = extractVideo(video || streamingPlaylist)
|
||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid)
|
||||
|
||||
try {
|
||||
await updateTorrentMetadata(video || streamingPlaylist, file)
|
||||
|
||||
await file.save()
|
||||
} finally {
|
||||
fileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
async function loadVideoOrLog (videoId: number) {
|
||||
if (!videoId) return undefined
|
||||
|
||||
const video = await VideoModel.load(videoId)
|
||||
if (!video) {
|
||||
logger.debug('Do not process torrent for video %d: does not exist anymore.', videoId)
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
|
||||
if (!streamingPlaylistId) return undefined
|
||||
|
||||
const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId)
|
||||
if (!streamingPlaylist) {
|
||||
logger.debug('Do not process torrent for streaming playlist %d: does not exist anymore.', streamingPlaylistId)
|
||||
}
|
||||
|
||||
return streamingPlaylist
|
||||
}
|
||||
|
||||
async function loadFileOrLog (videoFileId: number) {
|
||||
if (!videoFileId) return undefined
|
||||
|
||||
const file = await VideoFileModel.load(videoFileId)
|
||||
|
||||
if (!file) {
|
||||
logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||
import {
|
||||
makeHLSFileAvailable,
|
||||
makeOriginalFileAvailable,
|
||||
makeWebVideoFileAvailable,
|
||||
removeHLSFileObjectStorageByFilename,
|
||||
removeHLSObjectStorage,
|
||||
removeOriginalFileObjectStorage,
|
||||
removeWebVideoObjectStorage
|
||||
} from '@server/lib/object-storage/index.js'
|
||||
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
|
||||
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { join } from 'path'
|
||||
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('move-file-system')
|
||||
|
||||
export async function processMoveToFileSystem (job: Job) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
logger.info('Moving video %s to file system in job %s.', payload.videoUUID, job.id)
|
||||
|
||||
await moveToJob({
|
||||
jobId: job.id,
|
||||
videoUUID: payload.videoUUID,
|
||||
loggerTags: lTagsBase().tags,
|
||||
|
||||
moveWebVideoFiles,
|
||||
moveHLSFiles,
|
||||
moveVideoSourceFile,
|
||||
|
||||
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
|
||||
moveToFailedState: moveToFailedMoveToFileSystemState
|
||||
})
|
||||
}
|
||||
|
||||
export async function onMoveToFileSystemFailure (job: Job, err: any) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
|
||||
await onMoveToStorageFailure({
|
||||
videoUUID: payload.videoUUID,
|
||||
err,
|
||||
lTags: lTagsBase(),
|
||||
moveToFailedState: moveToFailedMoveToFileSystemState
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveVideoSourceFile (source: MVideoSource) {
|
||||
if (source.storage === FileStorage.FILE_SYSTEM) return
|
||||
|
||||
await makeOriginalFileAvailable(
|
||||
source.keptOriginalFilename,
|
||||
VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
|
||||
)
|
||||
|
||||
const oldFileUrl = source.fileUrl
|
||||
|
||||
source.fileUrl = null
|
||||
source.storage = FileStorage.FILE_SYSTEM
|
||||
await source.save()
|
||||
|
||||
logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
|
||||
|
||||
await removeOriginalFileObjectStorage(source)
|
||||
}
|
||||
|
||||
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||
for (const file of video.VideoFiles) {
|
||||
if (file.storage === FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
|
||||
await onVideoFileMoved({
|
||||
videoOrPlaylist: video,
|
||||
file,
|
||||
objetStorageRemover: () => removeWebVideoObjectStorage(file)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||
for (const playlist of video.VideoStreamingPlaylists) {
|
||||
const playlistWithVideo = playlist.withVideo(video)
|
||||
|
||||
for (const file of playlist.VideoFiles) {
|
||||
if (file.storage === FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
// Resolution playlist
|
||||
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
|
||||
await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
|
||||
await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
|
||||
|
||||
await onVideoFileMoved({
|
||||
videoOrPlaylist: playlistWithVideo,
|
||||
file,
|
||||
objetStorageRemover: async () => {
|
||||
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlistFilename)
|
||||
await removeHLSFileObjectStorageByFilename(playlistWithVideo, file.filename)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function onVideoFileMoved (options: {
|
||||
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||
file: MVideoFile
|
||||
objetStorageRemover: () => Promise<any>
|
||||
}) {
|
||||
const { videoOrPlaylist, file, objetStorageRemover } = options
|
||||
|
||||
const oldFileUrl = file.fileUrl
|
||||
|
||||
file.fileUrl = null
|
||||
file.storage = FileStorage.FILE_SYSTEM
|
||||
|
||||
await updateTorrentMetadata(videoOrPlaylist, file)
|
||||
await file.save()
|
||||
|
||||
logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
|
||||
await objetStorageRemover()
|
||||
}
|
||||
|
||||
async function doAfterLastMove (options: {
|
||||
video: MVideoWithAllFiles
|
||||
previousVideoState: VideoStateType
|
||||
isNewVideo: boolean
|
||||
}) {
|
||||
const { video, previousVideoState, isNewVideo } = options
|
||||
|
||||
for (const playlist of video.VideoStreamingPlaylists) {
|
||||
if (playlist.storage === FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
const playlistWithVideo = playlist.withVideo(video)
|
||||
|
||||
for (const filename of [ playlist.playlistFilename, playlist.segmentsSha256Filename ]) {
|
||||
await makeHLSFileAvailable(playlistWithVideo, filename, join(getHLSDirectory(video), filename))
|
||||
}
|
||||
|
||||
playlist.playlistUrl = null
|
||||
playlist.segmentsSha256Url = null
|
||||
playlist.storage = FileStorage.FILE_SYSTEM
|
||||
|
||||
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
||||
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
|
||||
|
||||
await playlist.save()
|
||||
|
||||
await removeHLSObjectStorage(playlistWithVideo)
|
||||
}
|
||||
|
||||
await moveToNextState({ video, previousVideoState, isNewVideo })
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||
import { storeHLSFileFromFilename, storeOriginalVideoFile, storeWebVideoFile } from '@server/lib/object-storage/index.js'
|
||||
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
|
||||
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { join } from 'path'
|
||||
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('move-object-storage')
|
||||
|
||||
export async function processMoveToObjectStorage (job: Job) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
logger.info('Moving video %s to object storage in job %s.', payload.videoUUID, job.id)
|
||||
|
||||
await moveToJob({
|
||||
jobId: job.id,
|
||||
videoUUID: payload.videoUUID,
|
||||
loggerTags: lTagsBase().tags,
|
||||
|
||||
moveWebVideoFiles,
|
||||
moveHLSFiles,
|
||||
moveVideoSourceFile,
|
||||
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
|
||||
moveToFailedState: moveToFailedMoveToObjectStorageState
|
||||
})
|
||||
}
|
||||
|
||||
export async function onMoveToObjectStorageFailure (job: Job, err: any) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
|
||||
await onMoveToStorageFailure({
|
||||
videoUUID: payload.videoUUID,
|
||||
err,
|
||||
lTags: lTagsBase(),
|
||||
moveToFailedState: moveToFailedMoveToObjectStorageState
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveVideoSourceFile (source: MVideoSource) {
|
||||
if (source.storage !== FileStorage.FILE_SYSTEM) return
|
||||
|
||||
const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
|
||||
const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
|
||||
|
||||
source.storage = FileStorage.OBJECT_STORAGE
|
||||
source.fileUrl = fileUrl
|
||||
await source.save()
|
||||
|
||||
logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
|
||||
|
||||
await remove(sourcePath)
|
||||
}
|
||||
|
||||
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||
for (const file of video.VideoFiles) {
|
||||
if (file.storage !== FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
const fileUrl = await storeWebVideoFile(video, file)
|
||||
|
||||
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
|
||||
await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
|
||||
}
|
||||
}
|
||||
|
||||
async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||
for (const playlist of video.VideoStreamingPlaylists) {
|
||||
const playlistWithVideo = playlist.withVideo(video)
|
||||
|
||||
for (const file of playlist.VideoFiles) {
|
||||
if (file.storage !== FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
// Resolution playlist
|
||||
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
|
||||
await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
|
||||
|
||||
// Resolution fragmented file
|
||||
const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
|
||||
|
||||
const oldPath = join(getHLSDirectory(video), file.filename)
|
||||
|
||||
await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function onVideoFileMoved (options: {
|
||||
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||
file: MVideoFile
|
||||
fileUrl: string
|
||||
oldPath: string
|
||||
}) {
|
||||
const { videoOrPlaylist, file, fileUrl, oldPath } = options
|
||||
|
||||
file.fileUrl = fileUrl
|
||||
file.storage = FileStorage.OBJECT_STORAGE
|
||||
|
||||
await updateTorrentMetadata(videoOrPlaylist, file)
|
||||
await file.save()
|
||||
|
||||
logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
|
||||
await remove(oldPath)
|
||||
}
|
||||
|
||||
async function doAfterLastMove (options: {
|
||||
video: MVideoWithAllFiles
|
||||
previousVideoState: VideoStateType
|
||||
isNewVideo: boolean
|
||||
}) {
|
||||
const { video, previousVideoState, isNewVideo } = options
|
||||
|
||||
for (const playlist of video.VideoStreamingPlaylists) {
|
||||
if (playlist.storage === FileStorage.OBJECT_STORAGE) continue
|
||||
|
||||
const playlistWithVideo = playlist.withVideo(video)
|
||||
|
||||
playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
|
||||
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
|
||||
playlist.storage = FileStorage.OBJECT_STORAGE
|
||||
|
||||
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
||||
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
|
||||
|
||||
await playlist.save()
|
||||
}
|
||||
|
||||
await remove(getHLSDirectory(video))
|
||||
await moveToNextState({ video, previousVideoState, isNewVideo })
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { Notifier } from '@server/lib/notifier/index.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { NotifyPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processNotify (job: Job) {
|
||||
const payload = job.data as NotifyPayload
|
||||
logger.info('Processing %s notification in job %s.', payload.action, job.id)
|
||||
|
||||
if (payload.action === 'new-video') return doNotifyNewVideo(payload)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processNotify
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function doNotifyNewVideo (payload: NotifyPayload & { action: 'new-video' }) {
|
||||
const refreshedVideo = await VideoModel.loadFull(payload.videoUUID)
|
||||
if (!refreshedVideo) return
|
||||
|
||||
Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(refreshedVideo)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
|
||||
export async function moveToJob (options: {
|
||||
jobId: string
|
||||
videoUUID: string
|
||||
loggerTags: (number | string)[]
|
||||
|
||||
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||
moveVideoSourceFile: (source: MVideoSource) => Promise<void>
|
||||
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
|
||||
doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
|
||||
}) {
|
||||
const {
|
||||
jobId,
|
||||
loggerTags,
|
||||
videoUUID,
|
||||
moveVideoSourceFile,
|
||||
moveHLSFiles,
|
||||
moveWebVideoFiles,
|
||||
moveToFailedState,
|
||||
doAfterLastMove
|
||||
} = options
|
||||
|
||||
const lTagsBase = loggerTagsFactory(...loggerTags)
|
||||
|
||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoUUID)
|
||||
|
||||
const video = await VideoModel.loadWithFiles(videoUUID)
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info('Can\'t process job %d, video does not exist.', jobId, lTagsBase(videoUUID))
|
||||
fileMutexReleaser()
|
||||
return undefined
|
||||
}
|
||||
|
||||
const lTags = lTagsBase(video.uuid, video.url)
|
||||
|
||||
try {
|
||||
const source = await VideoSourceModel.loadLatest(video.id)
|
||||
if (source) {
|
||||
logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
|
||||
|
||||
await moveVideoSourceFile(source)
|
||||
}
|
||||
|
||||
if (video.VideoFiles) {
|
||||
logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
|
||||
|
||||
await moveWebVideoFiles(video)
|
||||
}
|
||||
|
||||
if (video.VideoStreamingPlaylists) {
|
||||
logger.debug('Moving HLS playlist of %s.', video.uuid, lTags)
|
||||
|
||||
await moveHLSFiles(video)
|
||||
}
|
||||
|
||||
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
|
||||
if (pendingMove === 0) {
|
||||
logger.info('Running cleanup after moving files (video %s in job %s)', video.uuid, jobId, lTags)
|
||||
|
||||
await doAfterLastMove(video)
|
||||
}
|
||||
} catch (err) {
|
||||
await onMoveToStorageFailure({ videoUUID, err, lTags, moveToFailedState })
|
||||
|
||||
throw err
|
||||
} finally {
|
||||
fileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
export async function onMoveToStorageFailure (options: {
|
||||
videoUUID: string
|
||||
err: any
|
||||
lTags: LoggerTags
|
||||
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
|
||||
}) {
|
||||
const { videoUUID, err, lTags, moveToFailedState } = options
|
||||
|
||||
const video = await VideoModel.loadWithFiles(videoUUID)
|
||||
if (!video) return
|
||||
|
||||
logger.error('Cannot move video %s storage.', video.url, { err, ...lTags })
|
||||
|
||||
await moveToFailedState(video)
|
||||
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { TranscodingJobBuilderPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { JobQueue } from '../job-queue.js'
|
||||
|
||||
async function processTranscodingJobBuilder (job: Job) {
|
||||
const payload = job.data as TranscodingJobBuilderPayload
|
||||
|
||||
logger.info('Processing transcoding job builder in job %s.', job.id)
|
||||
|
||||
if (payload.optimizeJob) {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
const user = await UserModel.loadByVideoId(video.id)
|
||||
const videoFile = video.getMaxQualityFile()
|
||||
|
||||
await createOptimizeOrMergeAudioJobs({
|
||||
...pick(payload.optimizeJob, [ 'isNewVideo' ]),
|
||||
|
||||
video,
|
||||
videoFile,
|
||||
user,
|
||||
videoFileAlreadyLocked: false
|
||||
})
|
||||
}
|
||||
|
||||
for (const job of (payload.jobs || [])) {
|
||||
await JobQueue.Instance.createJob(job)
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
|
||||
}
|
||||
|
||||
for (const sequentialJobs of (payload.sequentialJobs || [])) {
|
||||
await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs)
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.filter(s => !!s).length)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processTranscodingJobBuilder
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { logger } from '@server/helpers/logger.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { synchronizeChannel } from '@server/lib/sync-channel.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
|
||||
import { MChannelSync } from '@server/types/models/index.js'
|
||||
import { VideoChannelImportPayload } from '@peertube/peertube-models'
|
||||
|
||||
export async function processVideoChannelImport (job: Job) {
|
||||
const payload = job.data as VideoChannelImportPayload
|
||||
|
||||
logger.info('Processing video channel import in job %s.', job.id)
|
||||
|
||||
// Channel import requires only http upload to be allowed
|
||||
if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
|
||||
throw new Error('Cannot import channel as the HTTP upload is disabled')
|
||||
}
|
||||
|
||||
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
|
||||
throw new Error('Cannot import channel as the synchronization is disabled')
|
||||
}
|
||||
|
||||
let channelSync: MChannelSync
|
||||
if (payload.partOfChannelSyncId) {
|
||||
channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId)
|
||||
|
||||
if (!channelSync) {
|
||||
throw new Error('Unknown channel sync specified in videos channel import')
|
||||
}
|
||||
}
|
||||
|
||||
const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
|
||||
|
||||
logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
|
||||
|
||||
await synchronizeChannel({
|
||||
channel: videoChannel,
|
||||
externalChannelUrl: payload.externalChannelUrl,
|
||||
channelSync,
|
||||
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { copy } from 'fs-extra/esm'
|
||||
import { VideoFileImportPayload } from '@peertube/peertube-models'
|
||||
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { JobQueue } from '../job-queue.js'
|
||||
import { buildMoveJob } from '@server/lib/video-jobs.js'
|
||||
import { buildNewFile } from '@server/lib/video-file.js'
|
||||
|
||||
async function processVideoFileImport (job: Job) {
|
||||
const payload = job.data as VideoFileImportPayload
|
||||
logger.info('Processing video file import in job %s.', job.id)
|
||||
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info('Do not process job %d, video does not exist.', job.id)
|
||||
return undefined
|
||||
}
|
||||
|
||||
await updateVideoFile(video, payload.filePath)
|
||||
|
||||
if (CONFIG.OBJECT_STORAGE.ENABLED) {
|
||||
await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState: video.state, type: 'move-to-object-storage' }))
|
||||
} else {
|
||||
await federateVideoIfNeeded(video, false)
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoFileImport
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(inputFilePath)
|
||||
const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === resolution)
|
||||
|
||||
if (currentVideoFile) {
|
||||
// Remove old file and old torrent
|
||||
await video.removeWebVideoFile(currentVideoFile)
|
||||
// Remove the old video file from the array
|
||||
video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
|
||||
|
||||
await currentVideoFile.destroy()
|
||||
}
|
||||
|
||||
const newVideoFile = await buildNewFile({ mode: 'web-video', path: inputFilePath })
|
||||
newVideoFile.videoId = video.id
|
||||
|
||||
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
|
||||
await copy(inputFilePath, outputPath)
|
||||
|
||||
video.VideoFiles.push(newVideoFile)
|
||||
await createTorrentAndSetInfoHash(video, newVideoFile)
|
||||
|
||||
await newVideoFile.save()
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
import { buildAspectRatio } from '@peertube/peertube-core-utils'
|
||||
import {
|
||||
ffprobePromise,
|
||||
getChaptersFromContainer, getVideoStreamDuration
|
||||
} from '@peertube/peertube-ffmpeg'
|
||||
import {
|
||||
ThumbnailType,
|
||||
ThumbnailType_Type,
|
||||
VideoImportPayload,
|
||||
VideoImportPreventExceptionResult,
|
||||
VideoImportState,
|
||||
VideoImportTorrentPayload,
|
||||
VideoImportTorrentPayloadType,
|
||||
VideoImportYoutubeDLPayload,
|
||||
VideoImportYoutubeDLPayloadType, VideoState
|
||||
} from '@peertube/peertube-models'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
|
||||
import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'
|
||||
import { setAndSaveVideoAutomaticTags } from '@server/lib/automatic-tags/automatic-tags.js'
|
||||
import { isPostImportVideoAccepted } from '@server/lib/moderation.js'
|
||||
import { Hooks } from '@server/lib/plugins/hooks.js'
|
||||
import { ServerConfigManager } from '@server/lib/server-config-manager.js'
|
||||
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
|
||||
import { isUserQuotaValid } from '@server/lib/user.js'
|
||||
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
|
||||
import { replaceChaptersIfNotExist } from '@server/lib/video-chapters.js'
|
||||
import { buildNewFile } from '@server/lib/video-file.js'
|
||||
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { buildNextVideoState } from '@server/lib/video-state.js'
|
||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { move, remove } from 'fs-extra/esm'
|
||||
import { stat } from 'fs/promises'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { getSecureTorrentName } from '../../../helpers/utils.js'
|
||||
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent.js'
|
||||
import { CONSTRAINTS_FIELDS, JOB_TTL } from '../../../initializers/constants.js'
|
||||
import { sequelizeTypescript } from '../../../initializers/database.js'
|
||||
import { VideoFileModel } from '../../../models/video/video-file.js'
|
||||
import { VideoImportModel } from '../../../models/video/video-import.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
import { federateVideoIfNeeded } from '../../activitypub/videos/index.js'
|
||||
import { Notifier } from '../../notifier/index.js'
|
||||
import { generateLocalVideoMiniature } from '../../thumbnail.js'
|
||||
import { JobQueue } from '../job-queue.js'
|
||||
|
||||
async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
|
||||
const payload = job.data as VideoImportPayload
|
||||
|
||||
const videoImport = await getVideoImportOrDie(payload)
|
||||
if (videoImport.state === VideoImportState.CANCELLED) {
|
||||
logger.info('Do not process import since it has been cancelled', { payload })
|
||||
return { resultType: 'success' }
|
||||
}
|
||||
|
||||
videoImport.state = VideoImportState.PROCESSING
|
||||
await videoImport.save()
|
||||
|
||||
try {
|
||||
if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
|
||||
if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
|
||||
|
||||
return { resultType: 'success' }
|
||||
} catch (err) {
|
||||
if (!payload.preventException) throw err
|
||||
|
||||
logger.warn('Catch error in video import to send value to parent job.', { payload, err })
|
||||
return { resultType: 'error' }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoImport
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) {
|
||||
logger.info('Processing torrent video import in job %s.', job.id)
|
||||
|
||||
const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: payload.videoImportId }
|
||||
|
||||
const target = {
|
||||
torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
|
||||
uri: videoImport.magnetUri
|
||||
}
|
||||
return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options)
|
||||
}
|
||||
|
||||
async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) {
|
||||
logger.info('Processing youtubeDL video import in job %s.', job.id)
|
||||
|
||||
const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: videoImport.id }
|
||||
|
||||
const youtubeDL = new YoutubeDLWrapper(
|
||||
videoImport.targetUrl,
|
||||
ServerConfigManager.Instance.getEnabledResolutions('vod'),
|
||||
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
|
||||
)
|
||||
|
||||
return processFile(
|
||||
() => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),
|
||||
videoImport,
|
||||
options
|
||||
)
|
||||
}
|
||||
|
||||
async function getVideoImportOrDie (payload: VideoImportPayload) {
|
||||
const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
|
||||
if (!videoImport?.Video) {
|
||||
throw new Error(`Cannot import video ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`)
|
||||
}
|
||||
|
||||
return videoImport
|
||||
}
|
||||
|
||||
type ProcessFileOptions = {
|
||||
type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType
|
||||
generateTranscription: boolean
|
||||
videoImportId: number
|
||||
}
|
||||
async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
|
||||
let tmpVideoPath: string
|
||||
let videoFile: VideoFileModel
|
||||
|
||||
try {
|
||||
// Download video from youtubeDL
|
||||
tmpVideoPath = await downloader()
|
||||
|
||||
// Get information about this video
|
||||
const stats = await stat(tmpVideoPath)
|
||||
const isAble = await isUserQuotaValid({ userId: videoImport.User.id, uploadSize: stats.size })
|
||||
if (isAble === false) {
|
||||
throw new Error('The user video quota is exceeded with this video to import.')
|
||||
}
|
||||
|
||||
const ffprobe = await ffprobePromise(tmpVideoPath)
|
||||
const duration = await getVideoStreamDuration(tmpVideoPath, ffprobe)
|
||||
|
||||
const containerChapters = await getChaptersFromContainer({
|
||||
path: tmpVideoPath,
|
||||
maxTitleLength: CONSTRAINTS_FIELDS.VIDEO_CHAPTERS.TITLE.max,
|
||||
ffprobe
|
||||
})
|
||||
|
||||
videoFile = await buildNewFile({ mode: 'web-video', ffprobe, path: tmpVideoPath })
|
||||
videoFile.videoId = videoImport.videoId
|
||||
|
||||
const hookName = options.type === 'youtube-dl'
|
||||
? 'filter:api.video.post-import-url.accept.result'
|
||||
: 'filter:api.video.post-import-torrent.accept.result'
|
||||
|
||||
// Check we accept this video
|
||||
const acceptParameters = {
|
||||
videoImport,
|
||||
video: videoImport.Video,
|
||||
videoFilePath: tmpVideoPath,
|
||||
videoFile,
|
||||
user: videoImport.User
|
||||
}
|
||||
const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName)
|
||||
|
||||
if (acceptedResult.accepted !== true) {
|
||||
logger.info('Refused imported video.', { acceptedResult, acceptParameters })
|
||||
|
||||
videoImport.state = VideoImportState.REJECTED
|
||||
await videoImport.save()
|
||||
|
||||
throw new Error(acceptedResult.errorMessage)
|
||||
}
|
||||
|
||||
// Video is accepted, resuming preparation
|
||||
const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid)
|
||||
|
||||
try {
|
||||
const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile)
|
||||
|
||||
// Move file
|
||||
const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
|
||||
await move(tmpVideoPath, videoDestFile)
|
||||
|
||||
tmpVideoPath = null // This path is not used anymore
|
||||
|
||||
const thumbnails = await generateMiniature({ videoImportWithFiles, videoFile, ffprobe })
|
||||
|
||||
// Create torrent
|
||||
await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
|
||||
|
||||
const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
// Refresh video
|
||||
const video = await VideoModel.load(videoImportWithFiles.videoId, t)
|
||||
if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
|
||||
|
||||
await videoFile.save({ transaction: t })
|
||||
|
||||
// Update video DB object
|
||||
video.duration = duration
|
||||
video.state = buildNextVideoState(video.state)
|
||||
video.aspectRatio = buildAspectRatio({ width: videoFile.width, height: videoFile.height })
|
||||
await video.save({ transaction: t })
|
||||
|
||||
for (const thumbnail of thumbnails) {
|
||||
await video.addAndSaveThumbnail(thumbnail, t)
|
||||
}
|
||||
|
||||
await replaceChaptersIfNotExist({ video, chapters: containerChapters, transaction: t })
|
||||
|
||||
const automaticTags = await new AutomaticTagger().buildVideoAutomaticTags({ video, transaction: t })
|
||||
await setAndSaveVideoAutomaticTags({ video, automaticTags, transaction: t })
|
||||
|
||||
// Now we can federate the video (reload from database, we need more attributes)
|
||||
const videoForFederation = await VideoModel.loadFull(video.uuid, t)
|
||||
await federateVideoIfNeeded(videoForFederation, true, t)
|
||||
|
||||
// Update video import object
|
||||
videoImportWithFiles.state = VideoImportState.SUCCESS
|
||||
const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport
|
||||
|
||||
logger.info('Video %s imported.', video.uuid)
|
||||
|
||||
return { videoImportUpdated, video: videoForFederation }
|
||||
})
|
||||
})
|
||||
|
||||
await afterImportSuccess({
|
||||
videoImport: videoImportUpdated,
|
||||
video,
|
||||
videoFile,
|
||||
user: videoImport.User,
|
||||
generateTranscription: options.generateTranscription,
|
||||
videoFileAlreadyLocked: true
|
||||
})
|
||||
} finally {
|
||||
videoFileLockReleaser()
|
||||
}
|
||||
} catch (err) {
|
||||
await onImportError(err, tmpVideoPath, videoImport)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> {
|
||||
// Refresh video, privacy may have changed
|
||||
const video = await videoImport.Video.reload()
|
||||
const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
|
||||
|
||||
return Object.assign(videoImport, { Video: videoWithFiles })
|
||||
}
|
||||
|
||||
async function generateMiniature (options: {
|
||||
videoImportWithFiles: MVideoImportDefaultFiles
|
||||
videoFile: MVideoFile
|
||||
ffprobe: FfprobeData
|
||||
}) {
|
||||
const { ffprobe, videoFile, videoImportWithFiles } = options
|
||||
|
||||
const thumbnailsToGenerate: ThumbnailType_Type[] = []
|
||||
|
||||
if (!videoImportWithFiles.Video.getMiniature()) {
|
||||
thumbnailsToGenerate.push(ThumbnailType.MINIATURE)
|
||||
}
|
||||
|
||||
if (!videoImportWithFiles.Video.getPreview()) {
|
||||
thumbnailsToGenerate.push(ThumbnailType.PREVIEW)
|
||||
}
|
||||
|
||||
return generateLocalVideoMiniature({
|
||||
video: videoImportWithFiles.Video,
|
||||
videoFile,
|
||||
types: thumbnailsToGenerate,
|
||||
ffprobe
|
||||
})
|
||||
}
|
||||
|
||||
async function afterImportSuccess (options: {
|
||||
videoImport: MVideoImport
|
||||
video: MVideoFullLight
|
||||
videoFile: MVideoFile
|
||||
user: MUserId
|
||||
|
||||
videoFileAlreadyLocked: boolean
|
||||
|
||||
generateTranscription: boolean
|
||||
}) {
|
||||
const { video, videoFile, videoImport, user, generateTranscription, videoFileAlreadyLocked } = options
|
||||
|
||||
Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true })
|
||||
|
||||
if (video.isBlacklisted()) {
|
||||
const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
|
||||
|
||||
Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
|
||||
} else {
|
||||
Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(video)
|
||||
}
|
||||
|
||||
// Generate the storyboard in the job queue, and don't forget to federate an update after
|
||||
await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))
|
||||
|
||||
if (await VideoCaptionModel.hasVideoCaption(video.id) !== true && generateTranscription === true) {
|
||||
await createTranscriptionTaskIfNeeded(video)
|
||||
}
|
||||
|
||||
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
||||
await JobQueue.Instance.createJob(
|
||||
await buildMoveJob({ video, previousVideoState: VideoState.TO_IMPORT, type: 'move-to-object-storage' })
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
|
||||
await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user, videoFileAlreadyLocked })
|
||||
}
|
||||
}
|
||||
|
||||
async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) {
|
||||
try {
|
||||
if (tempVideoPath) await remove(tempVideoPath)
|
||||
} catch (errUnlink) {
|
||||
logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
|
||||
}
|
||||
|
||||
videoImport.error = err.message
|
||||
if (videoImport.state !== VideoImportState.REJECTED) {
|
||||
videoImport.state = VideoImportState.FAILED
|
||||
}
|
||||
await videoImport.save()
|
||||
|
||||
Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
|
||||
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@peertube/peertube-models'
|
||||
import { peertubeTruncate } from '@server/helpers/core-utils.js'
|
||||
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
|
||||
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live/index.js'
|
||||
import {
|
||||
generateHLSMasterPlaylistFilename,
|
||||
generateHlsSha256SegmentsFilename,
|
||||
getHLSDirectory,
|
||||
getLiveReplayBaseDirectory
|
||||
} from '@server/lib/paths.js'
|
||||
import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
|
||||
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js'
|
||||
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
|
||||
import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'
|
||||
import { moveToNextState } from '@server/lib/video-state.js'
|
||||
import { VideoBlacklistModel } from '@server/models/video/video-blacklist.js'
|
||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'
|
||||
import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js'
|
||||
import { VideoLiveModel } from '@server/models/video/video-live.js'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { readdir } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
import { JobQueue } from '../job-queue.js'
|
||||
|
||||
const lTags = loggerTagsFactory('live', 'job')
|
||||
|
||||
async function processVideoLiveEnding (job: Job) {
|
||||
const payload = job.data as VideoLiveEndingPayload
|
||||
|
||||
logger.info('Processing video live ending for %s.', payload.videoId, { payload, ...lTags() })
|
||||
|
||||
function logError () {
|
||||
logger.warn('Video live %d does not exist anymore. Cannot process live ending.', payload.videoId, lTags())
|
||||
}
|
||||
|
||||
const video = await VideoModel.load(payload.videoId)
|
||||
const live = await VideoLiveModel.loadByVideoId(payload.videoId)
|
||||
const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId)
|
||||
|
||||
if (!video || !live || !liveSession) {
|
||||
logError()
|
||||
return
|
||||
}
|
||||
|
||||
const permanentLive = live.permanentLive
|
||||
|
||||
liveSession.endingProcessed = true
|
||||
await liveSession.save()
|
||||
|
||||
if (liveSession.saveReplay !== true) {
|
||||
return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
|
||||
}
|
||||
|
||||
if (await hasReplayFiles(payload.replayDirectory) !== true) {
|
||||
logger.info(`No replay files found for live ${video.uuid}, skipping video replay creation.`, { ...lTags(video.uuid) })
|
||||
|
||||
return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
|
||||
}
|
||||
|
||||
if (permanentLive) {
|
||||
await saveReplayToExternalVideo({
|
||||
liveVideo: video,
|
||||
liveSession,
|
||||
publishedAt: payload.publishedAt,
|
||||
replayDirectory: payload.replayDirectory
|
||||
})
|
||||
|
||||
return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
|
||||
}
|
||||
|
||||
return replaceLiveByReplay({
|
||||
video,
|
||||
liveSession,
|
||||
live,
|
||||
permanentLive,
|
||||
replayDirectory: payload.replayDirectory
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoLiveEnding
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function saveReplayToExternalVideo (options: {
|
||||
liveVideo: MVideo
|
||||
liveSession: MVideoLiveSession
|
||||
publishedAt: string
|
||||
replayDirectory: string
|
||||
}) {
|
||||
const { liveVideo, liveSession, publishedAt, replayDirectory } = options
|
||||
|
||||
const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId)
|
||||
|
||||
const videoNameSuffix = ` - ${new Date(publishedAt).toLocaleString()}`
|
||||
const truncatedVideoName = peertubeTruncate(liveVideo.name, {
|
||||
length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max - videoNameSuffix.length
|
||||
})
|
||||
|
||||
const replayVideo = new VideoModel({
|
||||
name: truncatedVideoName + videoNameSuffix,
|
||||
isLive: false,
|
||||
state: VideoState.TO_TRANSCODE,
|
||||
duration: 0,
|
||||
|
||||
remote: liveVideo.remote,
|
||||
category: liveVideo.category,
|
||||
licence: liveVideo.licence,
|
||||
language: liveVideo.language,
|
||||
commentsPolicy: liveVideo.commentsPolicy,
|
||||
downloadEnabled: liveVideo.downloadEnabled,
|
||||
waitTranscoding: true,
|
||||
nsfw: liveVideo.nsfw,
|
||||
description: liveVideo.description,
|
||||
aspectRatio: liveVideo.aspectRatio,
|
||||
support: liveVideo.support,
|
||||
privacy: replaySettings.privacy,
|
||||
channelId: liveVideo.channelId
|
||||
}) as MVideoWithAllFiles
|
||||
|
||||
replayVideo.Thumbnails = []
|
||||
replayVideo.VideoFiles = []
|
||||
replayVideo.VideoStreamingPlaylists = []
|
||||
|
||||
replayVideo.url = getLocalVideoActivityPubUrl(replayVideo)
|
||||
|
||||
await replayVideo.save()
|
||||
|
||||
liveSession.replayVideoId = replayVideo.id
|
||||
await liveSession.save()
|
||||
|
||||
// If live is blacklisted, also blacklist the replay
|
||||
const blacklist = await VideoBlacklistModel.loadByVideoId(liveVideo.id)
|
||||
if (blacklist) {
|
||||
await VideoBlacklistModel.create({
|
||||
videoId: replayVideo.id,
|
||||
unfederated: blacklist.unfederated,
|
||||
reason: blacklist.reason,
|
||||
type: blacklist.type
|
||||
})
|
||||
}
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(liveVideo.uuid)
|
||||
|
||||
try {
|
||||
await assignReplayFilesToVideo({ video: replayVideo, replayDirectory })
|
||||
|
||||
await remove(replayDirectory)
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
|
||||
const thumbnails = await generateLocalVideoMiniature({
|
||||
video: replayVideo,
|
||||
videoFile: replayVideo.getMaxQualityFile(),
|
||||
types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ],
|
||||
ffprobe: undefined
|
||||
})
|
||||
|
||||
for (const thumbnail of thumbnails) {
|
||||
await replayVideo.addAndSaveThumbnail(thumbnail)
|
||||
}
|
||||
|
||||
await createStoryboardJob(replayVideo)
|
||||
await createTranscriptionTaskIfNeeded(replayVideo)
|
||||
|
||||
await moveToNextState({ video: replayVideo, isNewVideo: true })
|
||||
}
|
||||
|
||||
async function replaceLiveByReplay (options: {
|
||||
video: MVideo
|
||||
liveSession: MVideoLiveSession
|
||||
live: MVideoLive
|
||||
permanentLive: boolean
|
||||
replayDirectory: string
|
||||
}) {
|
||||
const { video: liveVideo, liveSession, live, permanentLive, replayDirectory } = options
|
||||
|
||||
const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId)
|
||||
const videoWithFiles = await VideoModel.loadFull(liveVideo.id)
|
||||
const hlsPlaylist = videoWithFiles.getHLSPlaylist()
|
||||
const replayInAnotherDirectory = isVideoInPublicDirectory(liveVideo.privacy) !== isVideoInPublicDirectory(replaySettings.privacy)
|
||||
|
||||
logger.info(`Replacing live ${liveVideo.uuid} by replay ${replayDirectory}.`, { replayInAnotherDirectory, ...lTags(liveVideo.uuid) })
|
||||
|
||||
await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist)
|
||||
|
||||
await live.destroy()
|
||||
|
||||
videoWithFiles.isLive = false
|
||||
videoWithFiles.privacy = replaySettings.privacy
|
||||
videoWithFiles.waitTranscoding = true
|
||||
videoWithFiles.state = VideoState.TO_TRANSCODE
|
||||
|
||||
await videoWithFiles.save()
|
||||
|
||||
liveSession.replayVideoId = videoWithFiles.id
|
||||
await liveSession.save()
|
||||
|
||||
await VideoFileModel.removeHLSFilesOfStreamingPlaylistId(hlsPlaylist.id)
|
||||
|
||||
// Reset playlist
|
||||
hlsPlaylist.VideoFiles = []
|
||||
hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename()
|
||||
hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename()
|
||||
await hlsPlaylist.save()
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoWithFiles.uuid)
|
||||
|
||||
try {
|
||||
await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
|
||||
|
||||
// Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay
|
||||
if (permanentLive) { // Remove session replay
|
||||
await remove(replayDirectory)
|
||||
} else {
|
||||
// We won't stream again in this live, we can delete the base replay directory
|
||||
await remove(getLiveReplayBaseDirectory(liveVideo))
|
||||
|
||||
// If the live was in another base directory, also delete it
|
||||
if (replayInAnotherDirectory) {
|
||||
await remove(getHLSDirectory(liveVideo))
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
|
||||
// Regenerate the thumbnail & preview?
|
||||
await regenerateMiniaturesIfNeeded(videoWithFiles, undefined)
|
||||
|
||||
// We consider this is a new video
|
||||
await moveToNextState({ video: videoWithFiles, isNewVideo: true })
|
||||
|
||||
await createStoryboardJob(videoWithFiles)
|
||||
await createTranscriptionTaskIfNeeded(videoWithFiles)
|
||||
}
|
||||
|
||||
async function assignReplayFilesToVideo (options: {
|
||||
video: MVideo
|
||||
replayDirectory: string
|
||||
}) {
|
||||
const { video, replayDirectory } = options
|
||||
|
||||
const concatenatedTsFiles = await readdir(replayDirectory)
|
||||
|
||||
logger.info(`Assigning replays ${replayDirectory} to video ${video.uuid}.`, { concatenatedTsFiles, ...lTags(video.uuid) })
|
||||
|
||||
for (const concatenatedTsFile of concatenatedTsFiles) {
|
||||
// Generating hls playlist can be long, reload the video in this case
|
||||
await video.reload()
|
||||
|
||||
const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
|
||||
|
||||
const probe = await ffprobePromise(concatenatedTsFilePath)
|
||||
const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
|
||||
const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe)
|
||||
|
||||
try {
|
||||
await generateHlsPlaylistResolutionFromTS({
|
||||
video,
|
||||
inputFileMutexReleaser: null, // Already locked in parent
|
||||
concatenatedTsFilePath,
|
||||
resolution,
|
||||
fps,
|
||||
isAAC: audioStream?.codec_name === 'aac'
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
|
||||
}
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
async function cleanupLiveAndFederate (options: {
|
||||
video: MVideo
|
||||
permanentLive: boolean
|
||||
streamingPlaylistId: number
|
||||
}) {
|
||||
const { permanentLive, video, streamingPlaylistId } = options
|
||||
|
||||
const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId)
|
||||
|
||||
if (streamingPlaylist) {
|
||||
if (permanentLive) {
|
||||
await cleanupAndDestroyPermanentLive(video, streamingPlaylist)
|
||||
} else {
|
||||
await cleanupUnsavedNormalLive(video, streamingPlaylist)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const fullVideo = await VideoModel.loadFull(video.id)
|
||||
return federateVideoIfNeeded(fullVideo, false, undefined)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
|
||||
}
|
||||
}
|
||||
|
||||
function createStoryboardJob (video: MVideo) {
|
||||
return JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))
|
||||
}
|
||||
|
||||
async function hasReplayFiles (replayDirectory: string) {
|
||||
return (await readdir(replayDirectory)).length !== 0
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler.js'
|
||||
import { VideoRedundancyPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processVideoRedundancy (job: Job) {
|
||||
const payload = job.data as VideoRedundancyPayload
|
||||
logger.info('Processing video redundancy in job %s.', job.id)
|
||||
|
||||
return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoRedundancy
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { join } from 'path'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
|
||||
import { isUserQuotaValid } from '@server/lib/user.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideo, MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { buildUUID } from '@peertube/peertube-node-utils'
|
||||
import { FFmpegEdition } from '@peertube/peertube-ffmpeg'
|
||||
import {
|
||||
VideoStudioEditionPayload,
|
||||
VideoStudioTask,
|
||||
VideoStudioTaskCutPayload,
|
||||
VideoStudioTaskIntroPayload,
|
||||
VideoStudioTaskOutroPayload,
|
||||
VideoStudioTaskPayload,
|
||||
VideoStudioTaskWatermarkPayload
|
||||
} from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('video-studio')
|
||||
|
||||
async function processVideoStudioEdition (job: Job) {
|
||||
const payload = job.data as VideoStudioEditionPayload
|
||||
const lTags = lTagsBase(payload.videoUUID)
|
||||
|
||||
logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
|
||||
|
||||
try {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
|
||||
|
||||
await safeCleanupStudioTMPFiles(payload.tasks)
|
||||
return undefined
|
||||
}
|
||||
|
||||
await checkUserQuotaOrThrow(video, payload)
|
||||
|
||||
const inputFile = video.getMaxQualityFile()
|
||||
|
||||
const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => {
|
||||
let tmpInputFilePath: string
|
||||
let outputPath: string
|
||||
|
||||
for (const task of payload.tasks) {
|
||||
const outputFilename = buildUUID() + inputFile.extname
|
||||
outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
|
||||
|
||||
await processTask({
|
||||
inputPath: tmpInputFilePath ?? originalFilePath,
|
||||
video,
|
||||
outputPath,
|
||||
task,
|
||||
lTags
|
||||
})
|
||||
|
||||
if (tmpInputFilePath) await remove(tmpInputFilePath)
|
||||
|
||||
// For the next iteration
|
||||
tmpInputFilePath = outputPath
|
||||
}
|
||||
|
||||
return outputPath
|
||||
})
|
||||
|
||||
logger.info('Video edition ended for video %s.', video.uuid, lTags)
|
||||
|
||||
await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks })
|
||||
} catch (err) {
|
||||
await safeCleanupStudioTMPFiles(payload.tasks)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoStudioEdition
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type TaskProcessorOptions <T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = {
|
||||
inputPath: string
|
||||
outputPath: string
|
||||
video: MVideo
|
||||
task: T
|
||||
lTags: { tags: (string | number)[] }
|
||||
}
|
||||
|
||||
const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = {
|
||||
'add-intro': processAddIntroOutro,
|
||||
'add-outro': processAddIntroOutro,
|
||||
'cut': processCut,
|
||||
'add-watermark': processAddWatermark
|
||||
}
|
||||
|
||||
async function processTask (options: TaskProcessorOptions) {
|
||||
const { video, task, lTags } = options
|
||||
|
||||
logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags })
|
||||
|
||||
const processor = taskProcessors[options.task.name]
|
||||
if (!process) throw new Error('Unknown task ' + task.name)
|
||||
|
||||
return processor(options)
|
||||
}
|
||||
|
||||
function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
|
||||
const { task, lTags } = options
|
||||
|
||||
logger.debug('Will add intro/outro to the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().addIntroOutro({
|
||||
...pick(options, [ 'inputPath', 'outputPath' ]),
|
||||
|
||||
introOutroPath: task.options.file,
|
||||
type: task.name === 'add-intro'
|
||||
? 'intro'
|
||||
: 'outro'
|
||||
})
|
||||
}
|
||||
|
||||
function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
|
||||
const { task, lTags } = options
|
||||
|
||||
logger.debug('Will cut the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().cutVideo({
|
||||
...pick(options, [ 'inputPath', 'outputPath' ]),
|
||||
|
||||
start: task.options.start,
|
||||
end: task.options.end
|
||||
})
|
||||
}
|
||||
|
||||
function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
|
||||
const { task, lTags } = options
|
||||
|
||||
logger.debug('Will add watermark to the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().addWatermark({
|
||||
...pick(options, [ 'inputPath', 'outputPath' ]),
|
||||
|
||||
watermarkPath: task.options.file,
|
||||
|
||||
videoFilters: {
|
||||
watermarkSizeRatio: task.options.watermarkSizeRatio,
|
||||
horitonzalMarginRatio: task.options.horitonzalMarginRatio,
|
||||
verticalMarginRatio: task.options.verticalMarginRatio
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) {
|
||||
const user = await UserModel.loadByVideoId(video.id)
|
||||
|
||||
const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file
|
||||
|
||||
const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder)
|
||||
if (await isUserQuotaValid({ userId: user.id, uploadSize: additionalBytes }) === false) {
|
||||
throw new Error('Quota exceeded for this user to edit the video')
|
||||
}
|
||||
}
|
||||
|
||||
function buildFFmpegEdition () {
|
||||
return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js'
|
||||
import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding.js'
|
||||
import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding.js'
|
||||
import { removeAllWebVideoFiles } from '@server/lib/video-file.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedTranscodingState } from '@server/lib/video-state.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||
import { MUser, MUserId, MVideoFullLight } from '@server/types/models/index.js'
|
||||
import {
|
||||
HLSTranscodingPayload,
|
||||
MergeAudioTranscodingPayload,
|
||||
NewWebVideoResolutionTranscodingPayload,
|
||||
OptimizeTranscodingPayload,
|
||||
VideoTranscodingPayload
|
||||
} from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
|
||||
type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
|
||||
|
||||
const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = {
|
||||
'new-resolution-to-hls': handleHLSJob,
|
||||
'new-resolution-to-web-video': handleNewWebVideoResolutionJob,
|
||||
'merge-audio-to-web-video': handleWebVideoMergeAudioJob,
|
||||
'optimize-to-web-video': handleWebVideoOptimizeJob
|
||||
}
|
||||
|
||||
const lTags = loggerTagsFactory('transcoding')
|
||||
|
||||
async function processVideoTranscoding (job: Job) {
|
||||
const payload = job.data as VideoTranscodingPayload
|
||||
logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID))
|
||||
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info('Do not process job %d, video does not exist.', job.id, lTags(payload.videoUUID))
|
||||
return undefined
|
||||
}
|
||||
|
||||
const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId)
|
||||
|
||||
const handler = handlers[payload.type]
|
||||
|
||||
if (!handler) {
|
||||
await moveToFailedTranscodingState(video)
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
|
||||
throw new Error('Cannot find transcoding handler for ' + payload.type)
|
||||
}
|
||||
|
||||
try {
|
||||
await handler(job, payload, video, user)
|
||||
} catch (error) {
|
||||
await moveToFailedTranscodingState(video)
|
||||
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
|
||||
throw error
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoTranscoding
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })
|
||||
|
||||
logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video })
|
||||
}
|
||||
|
||||
async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job })
|
||||
|
||||
logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) {
|
||||
logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job })
|
||||
|
||||
logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFullLight) {
|
||||
logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload })
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
|
||||
let video: MVideoFullLight
|
||||
|
||||
try {
|
||||
video = await VideoModel.loadFull(videoArg.uuid)
|
||||
|
||||
const videoFileInput = payload.copyCodecs
|
||||
? video.getWebVideoFile(payload.resolution)
|
||||
: video.getMaxQualityFile()
|
||||
|
||||
const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
|
||||
|
||||
await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
|
||||
return generateHlsPlaylistResolution({
|
||||
video,
|
||||
videoInputPath,
|
||||
inputFileMutexReleaser,
|
||||
resolution: payload.resolution,
|
||||
fps: payload.fps,
|
||||
copyCodecs: payload.copyCodecs,
|
||||
job
|
||||
})
|
||||
})
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
|
||||
logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
if (payload.deleteWebVideoFiles === true) {
|
||||
logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await removeAllWebVideoFiles(video)
|
||||
}
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { VideoTranscriptionPayload } from '@peertube/peertube-models'
|
||||
import { generateSubtitle } from '@server/lib/video-captions.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
|
||||
const lTags = loggerTagsFactory('transcription')
|
||||
|
||||
export async function processVideoTranscription (job: Job) {
|
||||
const payload = job.data as VideoTranscriptionPayload
|
||||
|
||||
logger.info('Processing video transcription in job %s.', job.id)
|
||||
|
||||
const video = await VideoModel.load(payload.videoUUID)
|
||||
if (!video) {
|
||||
logger.info('Do not process transcription job %d, video does not exist.', job.id, lTags(payload.videoUUID))
|
||||
return
|
||||
}
|
||||
|
||||
return generateSubtitle({ video })
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
|
||||
import { VideoViewModel } from '@server/models/view/video-view.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
import { Redis } from '../../redis.js'
|
||||
|
||||
async function processVideosViewsStats () {
|
||||
const lastHour = new Date()
|
||||
|
||||
// In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour
|
||||
if (!isTestOrDevInstance()) lastHour.setHours(lastHour.getHours() - 1)
|
||||
|
||||
const hour = lastHour.getHours()
|
||||
const startDate = lastHour.setMinutes(0, 0, 0)
|
||||
const endDate = lastHour.setMinutes(59, 59, 999)
|
||||
|
||||
const videoIds = await Redis.Instance.listVideosViewedForStats(hour)
|
||||
if (videoIds.length === 0) return
|
||||
|
||||
logger.info('Processing videos views stats in job for hour %d.', hour)
|
||||
|
||||
for (const videoId of videoIds) {
|
||||
try {
|
||||
const views = await Redis.Instance.getVideoViewsStats(videoId, hour)
|
||||
await Redis.Instance.deleteVideoViewsStats(videoId, hour)
|
||||
|
||||
if (views) {
|
||||
logger.debug('Adding %d views to video %d stats in hour %d.', views, videoId, hour)
|
||||
|
||||
try {
|
||||
const video = await VideoModel.load(videoId)
|
||||
if (!video) {
|
||||
logger.debug('Video %d does not exist anymore, skipping videos view stats.', videoId)
|
||||
continue
|
||||
}
|
||||
|
||||
await VideoViewModel.create({
|
||||
startDate: new Date(startDate),
|
||||
endDate: new Date(endDate),
|
||||
views,
|
||||
videoId
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error('Cannot create video views stats for video %d in hour %d.', videoId, hour, { err })
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error('Cannot update video views stats of video %d in hour %d.', videoId, hour, { err })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideosViewsStats
|
||||
}
|
||||
新しい課題から参照
ユーザをブロックする