はじまりの大地

このコミットが含まれているのは:
2024-07-15 09:14:04 +09:00
コミット 6632905f32
3501個のファイルの変更1439465行の追加0行の削除
+35
ファイルの表示
@@ -0,0 +1,35 @@
import Bluebird from 'bluebird'
import { logger } from '../../helpers/logger.js'
export abstract class AbstractScheduler {
protected abstract schedulerIntervalMs: number
private interval: NodeJS.Timeout
private isRunning = false
enable () {
if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.')
this.interval = setInterval(() => this.execute(), this.schedulerIntervalMs)
}
disable () {
clearInterval(this.interval)
}
async execute () {
if (this.isRunning === true) return
this.isRunning = true
try {
await this.internalExecute()
} catch (err) {
logger.error('Cannot execute %s scheduler.', this.constructor.name, { err })
} finally {
this.isRunning = false
}
}
protected abstract internalExecute (): Promise<any> | Bluebird<any>
}
+54
ファイルの表示
@@ -0,0 +1,54 @@
import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
import { logger } from '../../helpers/logger.js'
import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { ActorFollowModel } from '../../models/actor/actor-follow.js'
import { ActorFollowHealthCache } from '../actor-follow-health-cache.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class ActorFollowScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES
private constructor () {
super()
}
protected async internalExecute () {
await this.processPendingScores()
await this.removeBadActorFollows()
}
private async processPendingScores () {
const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore()
const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds()
const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds()
ActorFollowHealthCache.Instance.clearPendingFollowsScore()
ActorFollowHealthCache.Instance.clearBadFollowingServerIds()
ActorFollowHealthCache.Instance.clearGoodFollowingServerIds()
for (const inbox of Object.keys(pendingScores)) {
await ActorFollowModel.updateScore(inbox, pendingScores[inbox])
}
await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY)
await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS)
}
private async removeBadActorFollows () {
if (!isTestOrDevInstance()) logger.info('Removing bad actor follows (scheduler).')
try {
await ActorFollowModel.removeBadActorFollows()
} catch (err) {
logger.error('Error in bad actor follows scheduler.', { err })
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+75
ファイルの表示
@@ -0,0 +1,75 @@
import { doJSONRequest } from '@server/helpers/requests.js'
import { JobQueue } from '@server/lib/job-queue/index.js'
import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
import { getServerActor } from '@server/models/application/application.js'
import chunk from 'lodash-es/chunk.js'
import { logger } from '../../helpers/logger.js'
import { CONFIG } from '../../initializers/config.js'
import { SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class AutoFollowIndexInstances extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES
private lastCheck: Date
private constructor () {
super()
}
protected async internalExecute () {
return this.autoFollow()
}
private async autoFollow () {
if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return
const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL
logger.info('Auto follow instances of index %s.', indexUrl)
try {
const serverActor = await getServerActor()
const searchParams = { count: 1000 }
if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() })
this.lastCheck = new Date()
const { body } = await doJSONRequest<any>(indexUrl, { searchParams })
if (!body.data || Array.isArray(body.data) === false) {
logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body })
return
}
const hosts: string[] = body.data.map(o => o.host)
const chunks = chunk(hosts, 20)
for (const chunk of chunks) {
const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk)
for (const unfollowedHost of unfollowedHosts) {
const payload = {
host: unfollowedHost,
name: SERVER_ACTOR_NAME,
followerActorId: serverActor.id,
isAutoFollow: true
}
JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload })
}
}
} catch (err) {
logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err })
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+22
ファイルの表示
@@ -0,0 +1,22 @@
import { GeoIP } from '@server/helpers/geo-ip.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class GeoIPUpdateScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE
private constructor () {
super()
}
protected internalExecute () {
return GeoIP.Instance.updateDatabases()
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+54
ファイルの表示
@@ -0,0 +1,54 @@
import { doJSONRequest } from '@server/helpers/requests.js'
import { ApplicationModel } from '@server/models/application/application.js'
import { compareSemVer } from '@peertube/peertube-core-utils'
import { JoinPeerTubeVersions } from '@peertube/peertube-models'
import { logger } from '../../helpers/logger.js'
import { CONFIG } from '../../initializers/config.js'
import { PEERTUBE_VERSION, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { Notifier } from '../notifier/index.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class PeerTubeVersionCheckScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION
private constructor () {
super()
}
protected async internalExecute () {
return this.checkLatestVersion()
}
private async checkLatestVersion () {
if (CONFIG.PEERTUBE.CHECK_LATEST_VERSION.ENABLED === false) return
logger.info('Checking latest PeerTube version.')
const { body } = await doJSONRequest<JoinPeerTubeVersions>(CONFIG.PEERTUBE.CHECK_LATEST_VERSION.URL)
if (!body?.peertube?.latestVersion) {
logger.warn('Cannot check latest PeerTube version: body is invalid.', { body })
return
}
const latestVersion = body.peertube.latestVersion
const application = await ApplicationModel.load()
// Already checked this version
if (application.latestPeerTubeVersion === latestVersion) return
if (compareSemVer(PEERTUBE_VERSION, latestVersion) < 0) {
application.latestPeerTubeVersion = latestVersion
await application.save()
Notifier.Instance.notifyOfNewPeerTubeVersion(application, latestVersion)
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+74
ファイルの表示
@@ -0,0 +1,74 @@
import { compareSemVer } from '@peertube/peertube-core-utils'
import chunk from 'lodash-es/chunk.js'
import { logger } from '../../helpers/logger.js'
import { CONFIG } from '../../initializers/config.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { PluginModel } from '../../models/server/plugin.js'
import { Notifier } from '../notifier/index.js'
import { getLatestPluginsVersion } from '../plugins/plugin-index.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class PluginsCheckScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PLUGINS
private constructor () {
super()
}
protected async internalExecute () {
return this.checkLatestPluginsVersion()
}
private async checkLatestPluginsVersion () {
if (CONFIG.PLUGINS.INDEX.ENABLED === false) return
logger.info('Checking latest plugins version.')
const plugins = await PluginModel.listInstalled()
// Process 10 plugins in 1 HTTP request
const chunks = chunk(plugins, 10)
for (const chunk of chunks) {
// Find plugins according to their npm name
const pluginIndex: { [npmName: string]: PluginModel } = {}
for (const plugin of chunk) {
pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin
}
const npmNames = Object.keys(pluginIndex)
try {
const results = await getLatestPluginsVersion(npmNames)
for (const result of results) {
const plugin = pluginIndex[result.npmName]
if (!result.latestVersion) continue
if (
!plugin.latestVersion ||
(plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0)
) {
plugin.latestVersion = result.latestVersion
await plugin.save()
// Notify if there is an higher plugin version available
if (compareSemVer(plugin.version, result.latestVersion) < 0) {
Notifier.Instance.notifyOfNewPluginVersion(plugin)
}
logger.info('Plugin %s has a new latest version %s.', result.npmName, plugin.latestVersion)
}
}
} catch (err) {
logger.error('Cannot get latest plugins version.', { npmNames, err })
}
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+39
ファイルの表示
@@ -0,0 +1,39 @@
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants.js'
import { uploadx } from '../uploadx.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner')
export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
private lastExecutionTimeMs: number
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
private constructor () {
super()
this.lastExecutionTimeMs = new Date().getTime()
}
protected async internalExecute () {
logger.debug('Removing dangling resumable uploads', lTags())
const now = new Date().getTime()
try {
// Remove files that were not updated since the last execution
await uploadx.storage.purge(now - this.lastExecutionTimeMs)
} catch (error) {
logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() })
} finally {
this.lastExecutionTimeMs = now
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+30
ファイルの表示
@@ -0,0 +1,30 @@
import { logger } from '../../helpers/logger.js'
import { AbstractScheduler } from './abstract-scheduler.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { CONFIG } from '../../initializers/config.js'
import { UserExportModel } from '@server/models/user/user-export.js'
export class RemoveExpiredUserExportsScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_EXPIRED_USER_EXPORTS
private constructor () {
super()
}
protected async internalExecute () {
const expired = await UserExportModel.listExpired(CONFIG.EXPORT.USERS.EXPORT_EXPIRATION)
for (const userExport of expired) {
logger.info('Removing expired user exports ' + userExport.filename)
await userExport.destroy()
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+31
ファイルの表示
@@ -0,0 +1,31 @@
import { logger } from '../../helpers/logger.js'
import { AbstractScheduler } from './abstract-scheduler.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { UserVideoHistoryModel } from '../../models/user/user-video-history.js'
import { CONFIG } from '../../initializers/config.js'
export class RemoveOldHistoryScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY
private constructor () {
super()
}
protected internalExecute () {
if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
logger.info('Removing old videos history.')
const now = new Date()
const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
return UserVideoHistoryModel.removeOldHistory(beforeDate)
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+31
ファイルの表示
@@ -0,0 +1,31 @@
import { VideoViewModel } from '@server/models/view/video-view.js'
import { logger } from '../../helpers/logger.js'
import { CONFIG } from '../../initializers/config.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class RemoveOldViewsScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS
private constructor () {
super()
}
protected internalExecute () {
if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return
logger.info('Removing old videos views.')
const now = new Date()
const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString()
return VideoViewModel.removeOldRemoteViewsHistory(beforeDate)
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+46
ファイルの表示
@@ -0,0 +1,46 @@
import { CONFIG } from '@server/initializers/config.js'
import { RunnerJobModel } from '@server/models/runner/runner-job.js'
import { logger, loggerTagsFactory } from '../../helpers/logger.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { getRunnerJobHandlerClass } from '../runners/index.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('runner')
export class RunnerJobWatchDogScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG
private constructor () {
super()
}
protected async internalExecute () {
const vodStalledJobs = await RunnerJobModel.listStalledJobs({
staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD,
types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]
})
const liveStalledJobs = await RunnerJobModel.listStalledJobs({
staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE,
types: [ 'live-rtmp-hls-transcoding' ]
})
for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) {
logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type))
const Handler = getRunnerJobHandlerClass(stalled)
await new Handler().abort({
runnerJob: stalled,
abortNotSupportedErrorMessage: 'Stalled runner job'
})
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+90
ファイルの表示
@@ -0,0 +1,90 @@
import { VideoPrivacy, VideoPrivacyType, VideoState } from '@peertube/peertube-models'
import { VideoModel } from '@server/models/video/video.js'
import { MScheduleVideoUpdate } from '@server/types/models/index.js'
import { logger } from '../../helpers/logger.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { sequelizeTypescript } from '../../initializers/database.js'
import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update.js'
import { isNewVideoPrivacyForFederation } from '../activitypub/videos/federate.js'
import { Notifier } from '../notifier/index.js'
import { addVideoJobsAfterUpdate } from '../video-jobs.js'
import { VideoPathManager } from '../video-path-manager.js'
import { setVideoPrivacy } from '../video-privacy.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class UpdateVideosScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS
private constructor () {
super()
}
protected async internalExecute () {
return this.updateVideos()
}
private async updateVideos () {
if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
for (const schedule of schedules) {
const videoOnly = await VideoModel.load(schedule.videoId)
const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
try {
const { video, published } = await this.updateAVideo(schedule)
if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
} catch (err) {
logger.error('Cannot update video', { err })
}
mutexReleaser()
}
}
private async updateAVideo (schedule: MScheduleVideoUpdate) {
let oldPrivacy: VideoPrivacyType
let isNewVideoForFederation: boolean
let published = false
const video = await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.loadFull(schedule.videoId, t)
if (video.state === VideoState.TO_TRANSCODE) return null
logger.info('Executing scheduled video update on %s.', video.uuid)
if (schedule.privacy) {
isNewVideoForFederation = isNewVideoPrivacyForFederation(video.privacy, schedule.privacy)
oldPrivacy = video.privacy
setVideoPrivacy(video, schedule.privacy)
await video.save({ transaction: t })
if (oldPrivacy === VideoPrivacy.PRIVATE) {
published = true
}
}
await schedule.destroy({ transaction: t })
return video
})
if (!video) {
return { video, published: false }
}
await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideoForFederation, nameChanged: false })
return { video, published }
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+50
ファイルの表示
@@ -0,0 +1,50 @@
import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { synchronizeChannel } from '../sync-channel.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
private constructor () {
super()
}
protected async internalExecute () {
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
logger.debug('Discard channels synchronization as the feature is disabled')
return
}
logger.info('Checking channels to synchronize')
const channelSyncs = await VideoChannelSyncModel.listSyncs()
for (const sync of channelSyncs) {
const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
logger.info(
'Creating video import jobs for "%s" sync with external channel "%s"',
channel.Actor.preferredUsername, sync.externalChannelUrl
)
const onlyAfter = sync.lastSyncAt || sync.createdAt
await synchronizeChannel({
channel,
externalChannelUrl: sync.externalChannelUrl,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
channelSync: sync,
onlyAfter
})
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+52
ファイルの表示
@@ -0,0 +1,52 @@
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoModel } from '@server/models/video/video.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { federateVideoIfNeeded } from '../activitypub/videos/index.js'
import { Redis } from '../redis.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('views')
export class VideoViewsBufferScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.VIDEO_VIEWS_BUFFER_UPDATE
private constructor () {
super()
}
protected async internalExecute () {
const videoIds = await Redis.Instance.listLocalVideosViewed()
if (videoIds.length === 0) return
for (const videoId of videoIds) {
try {
const views = await Redis.Instance.getLocalVideoViews(videoId)
await Redis.Instance.deleteLocalVideoViews(videoId)
const video = await VideoModel.loadFull(videoId)
if (!video) {
logger.debug('Video %d does not exist anymore, skipping videos view addition.', videoId, lTags())
continue
}
logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid))
// If this is a remote video, the origin instance will send us an update
await VideoModel.incrementViews(videoId, views)
// Send video update
video.views += views
await federateVideoIfNeeded(video, false)
} catch (err) {
logger.error('Cannot process local video views buffer of video %d.', videoId, { err, ...lTags() })
}
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
+375
ファイルの表示
@@ -0,0 +1,375 @@
import { move } from 'fs-extra/esm'
import { join } from 'path'
import { getServerActor } from '@server/models/application/application.js'
import { VideoModel } from '@server/models/video/video.js'
import {
MStreamingPlaylistFiles,
MVideoAccountLight,
MVideoFile,
MVideoFileVideo,
MVideoRedundancyFileVideo,
MVideoRedundancyStreamingPlaylistVideo,
MVideoRedundancyVideo,
MVideoWithAllFiles
} from '@server/types/models/index.js'
import { VideosRedundancyStrategy } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '../../helpers/logger.js'
import { downloadWebTorrentVideo } from '../../helpers/webtorrent.js'
import { CONFIG } from '../../initializers/config.js'
import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants.js'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy.js'
import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send/index.js'
import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url.js'
import { getOrCreateAPVideo } from '../activitypub/videos/index.js'
import { downloadPlaylistSegments } from '../hls.js'
import { removeVideoRedundancy } from '../redundancy.js'
import { generateHLSRedundancyUrl, generateWebVideoRedundancyUrl } from '../video-urls.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('redundancy')
type CandidateToDuplicate = {
redundancy: VideosRedundancyStrategy
video: MVideoWithAllFiles
files: MVideoFile[]
streamingPlaylists: MStreamingPlaylistFiles[]
}
function isMVideoRedundancyFileVideo (
o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo
): o is MVideoRedundancyFileVideo {
return !!(o as MVideoRedundancyFileVideo).VideoFile
}
export class VideosRedundancyScheduler extends AbstractScheduler {
private static instance: VideosRedundancyScheduler
protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
private constructor () {
super()
}
async createManualRedundancy (videoId: number) {
const videoToDuplicate = await VideoModel.loadWithFiles(videoId)
if (!videoToDuplicate) {
logger.warn('Video to manually duplicate %d does not exist anymore.', videoId, lTags())
return
}
return this.createVideoRedundancies({
video: videoToDuplicate,
redundancy: null,
files: videoToDuplicate.VideoFiles,
streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
})
}
protected async internalExecute () {
for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy, lTags())
try {
const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
if (!videoToDuplicate) continue
const candidateToDuplicate = {
video: videoToDuplicate,
redundancy: redundancyConfig,
files: videoToDuplicate.VideoFiles,
streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
}
await this.purgeCacheIfNeeded(candidateToDuplicate)
if (await this.isTooHeavy(candidateToDuplicate)) {
logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url, lTags(videoToDuplicate.uuid))
continue
}
logger.info(
'Will duplicate video %s in redundancy scheduler "%s".',
videoToDuplicate.url, redundancyConfig.strategy, lTags(videoToDuplicate.uuid)
)
await this.createVideoRedundancies(candidateToDuplicate)
} catch (err) {
logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err, ...lTags() })
}
}
await this.extendsLocalExpiration()
await this.purgeRemoteExpired()
}
static get Instance () {
return this.instance || (this.instance = new this())
}
private async extendsLocalExpiration () {
const expired = await VideoRedundancyModel.listLocalExpired()
for (const redundancyModel of expired) {
try {
const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
// If the admin disabled the redundancy, remove this redundancy instead of extending it
if (!redundancyConfig) {
logger.info(
'Destroying redundancy %s because the redundancy %s does not exist anymore.',
redundancyModel.url, redundancyModel.strategy
)
await removeVideoRedundancy(redundancyModel)
continue
}
const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy)
// If the admin decreased the cache size, remove this redundancy instead of extending it
if (totalUsed > redundancyConfig.size) {
logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
await removeVideoRedundancy(redundancyModel)
continue
}
await this.extendsRedundancy(redundancyModel)
} catch (err) {
logger.error(
'Cannot extend or remove expiration of %s video from our redundancy system.',
this.buildEntryLogId(redundancyModel), { err, ...lTags(redundancyModel.getVideoUUID()) }
)
}
}
}
private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) {
const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
// Redundancy strategy disabled, remove our redundancy instead of extending expiration
if (!redundancy) {
await removeVideoRedundancy(redundancyModel)
return
}
await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
}
private async purgeRemoteExpired () {
const expired = await VideoRedundancyModel.listRemoteExpired()
for (const redundancyModel of expired) {
try {
await removeVideoRedundancy(redundancyModel)
} catch (err) {
logger.error(
'Cannot remove redundancy %s from our redundancy system.',
this.buildEntryLogId(redundancyModel), lTags(redundancyModel.getVideoUUID())
)
}
}
}
private findVideoToDuplicate (cache: VideosRedundancyStrategy) {
if (cache.strategy === 'most-views') {
return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
}
if (cache.strategy === 'trending') {
return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
}
if (cache.strategy === 'recently-added') {
const minViews = cache.minViews
return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
}
}
private async createVideoRedundancies (data: CandidateToDuplicate) {
const video = await this.loadAndRefreshVideo(data.video.url)
if (!video) {
logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url, lTags(data.video.uuid))
return
}
for (const file of data.files) {
const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
if (existingRedundancy) {
await this.extendsRedundancy(existingRedundancy)
continue
}
await this.createVideoFileRedundancy(data.redundancy, video, file)
}
for (const streamingPlaylist of data.streamingPlaylists) {
const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
if (existingRedundancy) {
await this.extendsRedundancy(existingRedundancy)
continue
}
await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
}
}
private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) {
let strategy = 'manual'
let expiresOn: Date = null
if (redundancy) {
strategy = redundancy.strategy
expiresOn = this.buildNewExpiration(redundancy.minLifetime)
}
const file = fileArg as MVideoFileVideo
file.Video = video
const serverActor = await getServerActor()
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid))
const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT)
const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename)
await move(tmpPath, destPath, { overwrite: true })
const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
expiresOn,
url: getLocalVideoCacheFileActivityPubUrl(file),
fileUrl: generateWebVideoRedundancyUrl(file),
strategy,
videoFileId: file.id,
actorId: serverActor.id
})
createdModel.VideoFile = file
await sendCreateCacheFile(serverActor, video, createdModel)
logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url, lTags(video.uuid))
}
private async createStreamingPlaylistRedundancy (
redundancy: VideosRedundancyStrategy,
video: MVideoAccountLight,
playlistArg: MStreamingPlaylistFiles
) {
let strategy = 'manual'
let expiresOn: Date = null
if (redundancy) {
strategy = redundancy.strategy
expiresOn = this.buildNewExpiration(redundancy.minLifetime)
}
const playlist = Object.assign(playlistArg, { Video: video })
const serverActor = await getServerActor()
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid))
const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance
await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB)
const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
expiresOn,
url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
fileUrl: generateHLSRedundancyUrl(video, playlistArg),
strategy,
videoStreamingPlaylistId: playlist.id,
actorId: serverActor.id
})
createdModel.VideoStreamingPlaylist = playlist
await sendCreateCacheFile(serverActor, video, createdModel)
logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url, lTags(video.uuid))
}
private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) {
logger.info('Extending expiration of %s.', redundancy.url, lTags(redundancy.getVideoUUID()))
const serverActor = await getServerActor()
redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs)
await redundancy.save()
await sendUpdateCacheFile(serverActor, redundancy)
}
private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
while (await this.isTooHeavy(candidateToDuplicate)) {
const redundancy = candidateToDuplicate.redundancy
const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime)
if (!toDelete) return
const videoId = toDelete.VideoFile
? toDelete.VideoFile.videoId
: toDelete.VideoStreamingPlaylist.videoId
const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId)
for (const redundancy of redundancies) {
await removeVideoRedundancy(redundancy)
}
}
}
private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
const maxSize = candidateToDuplicate.redundancy.size
const { totalUsed: alreadyUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy)
const videoSize = this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)
const willUse = alreadyUsed + videoSize
logger.debug('Checking candidate size.', { maxSize, alreadyUsed, videoSize, willUse, ...lTags(candidateToDuplicate.video.uuid) })
return willUse > maxSize
}
private buildNewExpiration (expiresAfterMs: number) {
return new Date(Date.now() + expiresAfterMs)
}
private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) {
if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
}
private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number {
const fileReducer = (previous: number, current: MVideoFile) => previous + current.size
let allFiles = files
for (const p of playlists) {
allFiles = allFiles.concat(p.VideoFiles)
}
return allFiles.reduce(fileReducer, 0)
}
private async loadAndRefreshVideo (videoUrl: string) {
// We need more attributes and check if the video still exists
const getVideoOptions = {
videoObject: videoUrl,
syncParam: { rates: false, shares: false, comments: false, refreshVideo: true },
fetchType: 'all' as 'all'
}
const { video } = await getOrCreateAPVideo(getVideoOptions)
return video
}
}
+22
ファイルの表示
@@ -0,0 +1,22 @@
import { YoutubeDLCLI } from '@server/helpers/youtube-dl/index.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class YoutubeDlUpdateScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE
private constructor () {
super()
}
protected internalExecute () {
return YoutubeDLCLI.updateYoutubeDLBinary()
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}