はじまりの大地

server/core/helpers/activity-pub-utils.ts

@@ -0,0 +1,359 @@
+import { ContextType } from '@peertube/peertube-models'
+import { ACTIVITY_PUB, REMOTE_SCHEME } from '@server/initializers/constants.js'
+import { isArray } from './custom-validators/misc.js'
+import { buildDigest } from './peertube-crypto.js'
+import type { signJsonLDObject } from './peertube-jsonld.js'
+import { doJSONRequest } from './requests.js'
+
+export type ContextFilter = <T> (arg: T) => Promise<T>
+
+export function buildGlobalHTTPHeaders (
+  body: any,
+  digestBuilder: typeof buildDigest
+) {
+  return {
+    'digest': digestBuilder(body),
+    'content-type': 'application/activity+json',
+    'accept': ACTIVITY_PUB.ACCEPT_HEADER
+  }
+}
+
+export async function activityPubContextify <T> (data: T, type: ContextType, contextFilter: ContextFilter) {
+  return { ...await getContextData(type, contextFilter), ...data }
+}
+
+export async function signAndContextify <T> (options: {
+  byActor: { url: string, privateKey: string }
+  data: T
+  contextType: ContextType | null
+  contextFilter: ContextFilter
+  signerFunction: typeof signJsonLDObject<T>
+}) {
+  const { byActor, data, contextType, contextFilter, signerFunction } = options
+
+  const activity = contextType
+    ? await activityPubContextify(data, contextType, contextFilter)
+    : data
+
+  return signerFunction({ byActor, data: activity })
+}
+
+export async function getApplicationActorOfHost (host: string) {
+  const url = REMOTE_SCHEME.HTTP + '://' + host + '/.well-known/nodeinfo'
+  const { body } = await doJSONRequest<{ links: { rel: string, href: string }[] }>(url)
+
+  if (!isArray(body.links)) return undefined
+
+  const found = body.links.find(l => l.rel === 'https://www.w3.org/ns/activitystreams#Application')
+
+  return found?.href || undefined
+}
+
+export function getAPPublicValue (): 'https://www.w3.org/ns/activitystreams#Public' {
+  return 'https://www.w3.org/ns/activitystreams#Public'
+}
+
+export function hasAPPublic (toOrCC: string[]) {
+  if (!isArray(toOrCC)) return false
+
+  const publicValue = getAPPublicValue()
+
+  return toOrCC.some(f => f === 'as:Public' || publicValue)
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) }
+
+const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = {
+  Video: buildContext({
+    Hashtag: 'as:Hashtag',
+    category: 'sc:category',
+    licence: 'sc:license',
+    subtitleLanguage: 'sc:subtitleLanguage',
+    automaticallyGenerated: 'pt:automaticallyGenerated',
+    sensitive: 'as:sensitive',
+    language: 'sc:inLanguage',
+    identifier: 'sc:identifier',
+
+    isLiveBroadcast: 'sc:isLiveBroadcast',
+    liveSaveReplay: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:liveSaveReplay'
+    },
+    permanentLive: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:permanentLive'
+    },
+    latencyMode: {
+      '@type': 'sc:Number',
+      '@id': 'pt:latencyMode'
+    },
+
+    Infohash: 'pt:Infohash',
+
+    tileWidth: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileWidth'
+    },
+    tileHeight: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileHeight'
+    },
+    tileDuration: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileDuration'
+    },
+    aspectRatio: {
+      '@type': 'sc:Float',
+      '@id': 'pt:aspectRatio'
+    },
+
+    uuid: {
+      '@type': 'sc:identifier',
+      '@id': 'pt:uuid'
+    },
+
+    originallyPublishedAt: 'sc:datePublished',
+
+    uploadDate: 'sc:uploadDate',
+
+    hasParts: 'sc:hasParts',
+
+    views: {
+      '@type': 'sc:Number',
+      '@id': 'pt:views'
+    },
+    state: {
+      '@type': 'sc:Number',
+      '@id': 'pt:state'
+    },
+    size: {
+      '@type': 'sc:Number',
+      '@id': 'pt:size'
+    },
+    fps: {
+      '@type': 'sc:Number',
+      '@id': 'pt:fps'
+    },
+
+    // Keep for federation compatibility
+    commentsEnabled: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:commentsEnabled'
+    },
+
+    canReply: 'pt:canReply',
+    commentsPolicy: {
+      '@type': 'sc:Number',
+      '@id': 'pt:commentsPolicy'
+    },
+
+    downloadEnabled: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:downloadEnabled'
+    },
+    waitTranscoding: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:waitTranscoding'
+    },
+    support: {
+      '@type': 'sc:Text',
+      '@id': 'pt:support'
+    },
+    likes: {
+      '@id': 'as:likes',
+      '@type': '@id'
+    },
+    dislikes: {
+      '@id': 'as:dislikes',
+      '@type': '@id'
+    },
+    shares: {
+      '@id': 'as:shares',
+      '@type': '@id'
+    },
+    comments: {
+      '@id': 'as:comments',
+      '@type': '@id'
+    }
+  }),
+
+  Playlist: buildContext({
+    Playlist: 'pt:Playlist',
+    PlaylistElement: 'pt:PlaylistElement',
+    position: {
+      '@type': 'sc:Number',
+      '@id': 'pt:position'
+    },
+    startTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:startTimestamp'
+    },
+    stopTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:stopTimestamp'
+    },
+    uuid: {
+      '@type': 'sc:identifier',
+      '@id': 'pt:uuid'
+    }
+  }),
+
+  CacheFile: buildContext({
+    expires: 'sc:expires',
+    CacheFile: 'pt:CacheFile',
+    size: {
+      '@type': 'sc:Number',
+      '@id': 'pt:size'
+    },
+    fps: {
+      '@type': 'sc:Number',
+      '@id': 'pt:fps'
+    }
+  }),
+
+  Flag: buildContext({
+    Hashtag: 'as:Hashtag'
+  }),
+
+  Actor: buildContext({
+    playlists: {
+      '@id': 'pt:playlists',
+      '@type': '@id'
+    },
+    support: {
+      '@type': 'sc:Text',
+      '@id': 'pt:support'
+    },
+
+    lemmy: 'https://join-lemmy.org/ns#',
+    postingRestrictedToMods: 'lemmy:postingRestrictedToMods',
+
+    // TODO: remove in a few versions, introduced in 4.2
+    icons: 'as:icon'
+  }),
+
+  WatchAction: buildContext({
+    WatchAction: 'sc:WatchAction',
+    startTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:startTimestamp'
+    },
+    endTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:endTimestamp'
+    },
+    uuid: {
+      '@type': 'sc:identifier',
+      '@id': 'pt:uuid'
+    },
+    actionStatus: 'sc:actionStatus',
+    watchSections: {
+      '@type': '@id',
+      '@id': 'pt:watchSections'
+    },
+    addressRegion: 'sc:addressRegion',
+    addressCountry: 'sc:addressCountry'
+  }),
+
+  View: buildContext({
+    WatchAction: 'sc:WatchAction',
+    InteractionCounter: 'sc:InteractionCounter',
+    interactionType: 'sc:interactionType',
+    userInteractionCount: 'sc:userInteractionCount'
+  }),
+
+  Collection: buildContext(),
+  Follow: buildContext(),
+  Reject: buildContext(),
+  Accept: buildContext(),
+  Announce: buildContext(),
+
+  Comment: buildContext({
+    replyApproval: 'pt:replyApproval'
+  }),
+
+  Delete: buildContext(),
+  Rate: buildContext(),
+
+  ApproveReply: buildContext({
+    ApproveReply: 'pt:ApproveReply'
+  }),
+  RejectReply: buildContext({
+    RejectReply: 'pt:RejectReply'
+  }),
+
+  Chapters: buildContext({
+    hasPart: 'sc:hasPart',
+    endOffset: 'sc:endOffset',
+    startOffset: 'sc:startOffset'
+  })
+}
+
+let allContext: (string | ContextValue)[]
+export function getAllContext () {
+  if (allContext) return allContext
+
+  const processed = new Set<string>()
+  allContext = []
+
+  let staticContext: ContextValue = {}
+
+  for (const v of Object.values(contextStore)) {
+    for (const item of v) {
+      if (typeof item === 'string') {
+        if (!processed.has(item)) {
+          allContext.push(item)
+        }
+
+        processed.add(item)
+      } else {
+        for (const subKey of Object.keys(item)) {
+          if (!processed.has(subKey)) {
+            staticContext = { ...staticContext, [subKey]: item[subKey] }
+          }
+
+          processed.add(subKey)
+        }
+      }
+    }
+  }
+
+  allContext = [ ...allContext, staticContext ]
+
+  return allContext
+}
+
+async function getContextData (type: ContextType, contextFilter: ContextFilter) {
+  const contextData = contextFilter
+    ? await contextFilter(contextStore[type])
+    : contextStore[type]
+
+  return { '@context': contextData }
+}
+
+function buildContext (contextValue?: ContextValue) {
+  const baseContext = [
+    'https://www.w3.org/ns/activitystreams',
+    'https://w3id.org/security/v1',
+    {
+      RsaSignature2017: 'https://w3id.org/security#RsaSignature2017'
+    }
+  ]
+
+  if (!contextValue) return baseContext
+
+  return [
+    ...baseContext,
+
+    {
+      pt: 'https://joinpeertube.org/ns#',
+      sc: 'http://schema.org/',
+
+      ...contextValue
+    }
+  ]
+}

server/core/helpers/actors.ts

@@ -0,0 +1,22 @@
+import { ActivityPubActorType } from '@peertube/peertube-models'
+import { WEBSERVER } from '@server/initializers/constants.js'
+
+export function handleToNameAndHost (handle: string) {
+  let [ name, host ] = handle.split('@')
+  if (host === WEBSERVER.HOST) host = null
+
+  return { name, host, handle }
+}
+
+export function handlesToNameAndHost (handles: string[]) {
+  return handles.map(h => handleToNameAndHost(h))
+}
+
+const accountType = new Set([ 'Person', 'Application', 'Service', 'Organization' ])
+export function isAccountActor (type: ActivityPubActorType) {
+  return accountType.has(type)
+}
+
+export function isChannelActor (type: ActivityPubActorType) {
+  return type === 'Group'
+}

server/core/helpers/audit-logger.ts

@@ -0,0 +1,297 @@
+import {
+  AdminAbuse,
+  CustomConfig,
+  User,
+  VideoChannel,
+  VideoChannelSync,
+  VideoComment,
+  VideoDetails,
+  VideoImport
+} from '@peertube/peertube-models'
+import { AUDIT_LOG_FILENAME } from '@server/initializers/constants.js'
+import { diff } from 'deep-object-diff'
+import express from 'express'
+import { flatten } from 'flat'
+import { join } from 'path'
+import { addColors, config, createLogger, format, transports } from 'winston'
+import { CONFIG } from '../initializers/config.js'
+import { jsonLoggerFormat, labelFormatter } from './logger.js'
+
+function getAuditIdFromRes (res: express.Response) {
+  return res.locals.oauth.token.User.username
+}
+
+enum AUDIT_TYPE {
+  CREATE = 'create',
+  UPDATE = 'update',
+  DELETE = 'delete'
+}
+
+const colors = config.npm.colors
+colors.audit = config.npm.colors.info
+
+addColors(colors)
+
+const auditLogger = createLogger({
+  levels: { audit: 0 },
+  transports: [
+    new transports.File({
+      filename: join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME),
+      level: 'audit',
+      maxsize: 5242880,
+      maxFiles: 5,
+      format: format.combine(
+        format.timestamp(),
+        labelFormatter(),
+        format.splat(),
+        jsonLoggerFormat
+      )
+    })
+  ],
+  exitOnError: true
+})
+
+function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) {
+  let entityInfos: object
+
+  if (action === AUDIT_TYPE.UPDATE && oldEntity) {
+    const oldEntityKeys = oldEntity.toLogKeys()
+    const diffObject = diff(oldEntityKeys, entity.toLogKeys())
+    const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => {
+      newKeys[`new-${entry[0]}`] = entry[1]
+      return newKeys
+    }, {})
+    entityInfos = { ...oldEntityKeys, ...diffKeys }
+  } else {
+    entityInfos = { ...entity.toLogKeys() }
+  }
+
+  auditLogger.log('audit', JSON.stringify({
+    user,
+    domain,
+    action,
+    ...entityInfos
+  }))
+}
+
+function auditLoggerFactory (domain: string) {
+  return {
+    create (user: string, entity: EntityAuditView) {
+      auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity)
+    },
+    update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) {
+      auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity)
+    },
+    delete (user: string, entity: EntityAuditView) {
+      auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity)
+    }
+  }
+}
+
+abstract class EntityAuditView {
+  constructor (private readonly keysToKeep: Set<string>, private readonly prefix: string, private readonly entityInfos: object) { }
+
+  toLogKeys (): object {
+    const obj = flatten<object, any>(this.entityInfos, { delimiter: '-', safe: true })
+
+    return Object.keys(obj)
+      .filter(key => this.keysToKeep.has(key))
+      .reduce((p, k) => ({ ...p, [`${this.prefix}-${k}`]: obj[k] }), {})
+  }
+}
+
+const videoKeysToKeep = new Set([
+  'tags',
+  'uuid',
+  'id',
+  'uuid',
+  'createdAt',
+  'updatedAt',
+  'publishedAt',
+  'category',
+  'licence',
+  'language',
+  'privacy',
+  'description',
+  'duration',
+  'isLocal',
+  'name',
+  'thumbnailPath',
+  'previewPath',
+  'nsfw',
+  'waitTranscoding',
+  'account-id',
+  'account-uuid',
+  'account-name',
+  'channel-id',
+  'channel-uuid',
+  'channel-name',
+  'support',
+  'commentsPolicy',
+  'downloadEnabled'
+])
+class VideoAuditView extends EntityAuditView {
+  constructor (video: VideoDetails) {
+    super(videoKeysToKeep, 'video', video)
+  }
+}
+
+const videoImportKeysToKeep = new Set([
+  'id',
+  'targetUrl',
+  'video-name'
+])
+class VideoImportAuditView extends EntityAuditView {
+  constructor (videoImport: VideoImport) {
+    super(videoImportKeysToKeep, 'video-import', videoImport)
+  }
+}
+
+const commentKeysToKeep = new Set([
+  'id',
+  'text',
+  'threadId',
+  'inReplyToCommentId',
+  'videoId',
+  'createdAt',
+  'updatedAt',
+  'totalReplies',
+  'account-id',
+  'account-uuid',
+  'account-name'
+])
+class CommentAuditView extends EntityAuditView {
+  constructor (comment: VideoComment) {
+    super(commentKeysToKeep, 'comment', comment)
+  }
+}
+
+const userKeysToKeep = new Set([
+  'id',
+  'username',
+  'email',
+  'nsfwPolicy',
+  'autoPlayVideo',
+  'role',
+  'videoQuota',
+  'createdAt',
+  'account-id',
+  'account-uuid',
+  'account-name',
+  'account-followingCount',
+  'account-followersCount',
+  'account-createdAt',
+  'account-updatedAt',
+  'account-avatar-path',
+  'account-avatar-createdAt',
+  'account-avatar-updatedAt',
+  'account-displayName',
+  'account-description',
+  'videoChannels'
+])
+class UserAuditView extends EntityAuditView {
+  constructor (user: User) {
+    super(userKeysToKeep, 'user', user)
+  }
+}
+
+const channelKeysToKeep = new Set([
+  'id',
+  'uuid',
+  'name',
+  'followingCount',
+  'followersCount',
+  'createdAt',
+  'updatedAt',
+  'avatar-path',
+  'avatar-createdAt',
+  'avatar-updatedAt',
+  'displayName',
+  'description',
+  'support',
+  'isLocal',
+  'ownerAccount-id',
+  'ownerAccount-uuid',
+  'ownerAccount-name',
+  'ownerAccount-displayedName'
+])
+class VideoChannelAuditView extends EntityAuditView {
+  constructor (channel: VideoChannel) {
+    super(channelKeysToKeep, 'channel', channel)
+  }
+}
+
+const abuseKeysToKeep = new Set([
+  'id',
+  'reason',
+  'reporterAccount',
+  'createdAt'
+])
+class AbuseAuditView extends EntityAuditView {
+  constructor (abuse: AdminAbuse) {
+    super(abuseKeysToKeep, 'abuse', abuse)
+  }
+}
+
+const customConfigKeysToKeep = new Set([
+  'instance-name',
+  'instance-shortDescription',
+  'instance-description',
+  'instance-terms',
+  'instance-defaultClientRoute',
+  'instance-defaultNSFWPolicy',
+  'instance-customizations-javascript',
+  'instance-customizations-css',
+  'services-twitter-username',
+  'cache-previews-size',
+  'cache-captions-size',
+  'signup-enabled',
+  'signup-limit',
+  'signup-requiresEmailVerification',
+  'admin-email',
+  'user-videoQuota',
+  'transcoding-enabled',
+  'transcoding-threads',
+  'transcoding-resolutions'
+])
+class CustomConfigAuditView extends EntityAuditView {
+  constructor (customConfig: CustomConfig) {
+    const infos: any = customConfig
+    const resolutionsDict = infos.transcoding.resolutions
+    const resolutionsArray = []
+
+    Object.entries(resolutionsDict)
+          .forEach(([ resolution, isEnabled ]) => {
+            if (isEnabled) resolutionsArray.push(resolution)
+          })
+
+    Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } })
+    super(customConfigKeysToKeep, 'config', infos)
+  }
+}
+
+const channelSyncKeysToKeep = new Set([
+  'id',
+  'externalChannelUrl',
+  'channel-id',
+  'channel-name'
+])
+class VideoChannelSyncAuditView extends EntityAuditView {
+  constructor (channelSync: VideoChannelSync) {
+    super(channelSyncKeysToKeep, 'channelSync', channelSync)
+  }
+}
+
+export {
+  getAuditIdFromRes,
+
+  auditLoggerFactory,
+  VideoImportAuditView,
+  VideoChannelAuditView,
+  CommentAuditView,
+  UserAuditView,
+  VideoAuditView,
+  AbuseAuditView,
+  CustomConfigAuditView,
+  VideoChannelSyncAuditView
+}

server/core/helpers/captions-utils.ts

@@ -0,0 +1,52 @@
+import { createReadStream, createWriteStream } from 'fs'
+import { move, remove } from 'fs-extra/esm'
+import { Transform } from 'stream'
+import { MVideoCaption } from '@server/types/models/index.js'
+import { pipelinePromise } from './core-utils.js'
+
+async function moveAndProcessCaptionFile (physicalFile: { filename?: string, path: string }, videoCaption: MVideoCaption) {
+  const destination = videoCaption.getFSPath()
+
+  // Convert this srt file to vtt
+  if (physicalFile.path.endsWith('.srt')) {
+    await convertSrtToVtt(physicalFile.path, destination)
+    await remove(physicalFile.path)
+  } else if (physicalFile.path !== destination) { // Just move the vtt file
+    await move(physicalFile.path, destination, { overwrite: true })
+  }
+
+  // This is important in case if there is another attempt in the retry process
+  if (physicalFile.filename) physicalFile.filename = videoCaption.filename
+  physicalFile.path = destination
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  moveAndProcessCaptionFile
+}
+
+// ---------------------------------------------------------------------------
+
+async function convertSrtToVtt (source: string, destination: string) {
+  const fixVTT = new Transform({
+    transform: (chunk, _encoding, cb) => {
+      let block: string = chunk.toString()
+
+      block = block.replace(/(\d\d:\d\d:\d\d)(\s)/g, '$1.000$2')
+                   .replace(/(\d\d:\d\d:\d\d),(\d)(\s)/g, '$1.00$2$3')
+                   .replace(/(\d\d:\d\d:\d\d),(\d\d)(\s)/g, '$1.0$2$3')
+
+      return cb(undefined, block)
+    }
+  })
+
+  const srt2vtt = await import('srt-to-vtt')
+
+  return pipelinePromise(
+    createReadStream(source),
+    srt2vtt.default(),
+    fixVTT,
+    createWriteStream(destination)
+  )
+}

server/core/helpers/core-utils.ts

@@ -0,0 +1,298 @@
+/* eslint-disable no-useless-call */
+
+/*
+  Different from 'utils' because we don't import other PeerTube modules.
+  Useful to avoid circular dependencies.
+*/
+
+import { promisify1, promisify2, promisify3 } from '@peertube/peertube-core-utils'
+import { exec, ExecOptions } from 'child_process'
+import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
+import truncate from 'lodash-es/truncate.js'
+import { pipeline } from 'stream'
+import { URL } from 'url'
+import { promisify } from 'util'
+
+const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
+  if (!oldObject || typeof oldObject !== 'object') {
+    return valueConverter(oldObject)
+  }
+
+  if (Array.isArray(oldObject)) {
+    return oldObject.map(e => objectConverter(e, keyConverter, valueConverter))
+  }
+
+  const newObject = {}
+  Object.keys(oldObject).forEach(oldKey => {
+    const newKey = keyConverter(oldKey)
+    newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter)
+  })
+
+  return newObject
+}
+
+function mapToJSON (map: Map<any, any>) {
+  const obj: any = {}
+
+  for (const [ k, v ] of map) {
+    obj[k] = v
+  }
+
+  return obj
+}
+
+// ---------------------------------------------------------------------------
+
+const timeTable = {
+  ms: 1,
+  second: 1000,
+  minute: 60000,
+  hour: 3600000,
+  day: 3600000 * 24,
+  week: 3600000 * 24 * 7,
+  month: 3600000 * 24 * 30
+}
+
+export function parseDurationToMs (duration: number | string): number {
+  if (duration === null) return null
+  if (typeof duration === 'number') return duration
+  if (!isNaN(+duration)) return +duration
+
+  if (typeof duration === 'string') {
+    const split = duration.match(/^([\d.,]+)\s?(\w+)$/)
+
+    if (split.length === 3) {
+      const len = parseFloat(split[1])
+      let unit = split[2].replace(/s$/i, '').toLowerCase()
+      if (unit === 'm') {
+        unit = 'ms'
+      }
+
+      return (len || 1) * (timeTable[unit] || 0)
+    }
+  }
+
+  throw new Error(`Duration ${duration} could not be properly parsed`)
+}
+
+export function parseBytes (value: string | number): number {
+  if (typeof value === 'number') return value
+  if (!isNaN(+value)) return +value
+
+  const tgm = /^(\d+)\s*TB\s*(\d+)\s*GB\s*(\d+)\s*MB$/
+  const tg = /^(\d+)\s*TB\s*(\d+)\s*GB$/
+  const tm = /^(\d+)\s*TB\s*(\d+)\s*MB$/
+  const gm = /^(\d+)\s*GB\s*(\d+)\s*MB$/
+  const t = /^(\d+)\s*TB$/
+  const g = /^(\d+)\s*GB$/
+  const m = /^(\d+)\s*MB$/
+  const b = /^(\d+)\s*B$/
+
+  let match: RegExpMatchArray
+
+  if (value.match(tgm)) {
+    match = value.match(tgm)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024 * 1024 +
+      parseInt(match[3], 10) * 1024 * 1024
+  }
+
+  if (value.match(tg)) {
+    match = value.match(tg)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024 * 1024
+  }
+
+  if (value.match(tm)) {
+    match = value.match(tm)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024
+  }
+
+  if (value.match(gm)) {
+    match = value.match(gm)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024
+  }
+
+  if (value.match(t)) {
+    match = value.match(t)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024
+  }
+
+  if (value.match(g)) {
+    match = value.match(g)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024
+  }
+
+  if (value.match(m)) {
+    match = value.match(m)
+    return parseInt(match[1], 10) * 1024 * 1024
+  }
+
+  if (value.match(b)) {
+    match = value.match(b)
+    return parseInt(match[1], 10) * 1024
+  }
+
+  return parseInt(value, 10)
+}
+
+// ---------------------------------------------------------------------------
+
+function sanitizeUrl (url: string) {
+  const urlObject = new URL(url)
+
+  if (urlObject.protocol === 'https:' && urlObject.port === '443') {
+    urlObject.port = ''
+  } else if (urlObject.protocol === 'http:' && urlObject.port === '80') {
+    urlObject.port = ''
+  }
+
+  return urlObject.href.replace(/\/$/, '')
+}
+
+// Don't import remote scheme from constants because we are in core utils
+function sanitizeHost (host: string, remoteScheme: string) {
+  const toRemove = remoteScheme === 'https' ? 443 : 80
+
+  return host.replace(new RegExp(`:${toRemove}$`), '')
+}
+
+// ---------------------------------------------------------------------------
+
+// Consistent with .length, lodash truncate function is not
+function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) {
+  const truncatedStr = truncate(str, options)
+
+  // The truncated string is okay, we can return it
+  if (truncatedStr.length <= options.length) return truncatedStr
+
+  // Lodash takes into account all UTF characters, whereas String.prototype.length does not: some characters have a length of 2
+  // We always use the .length so we need to truncate more if needed
+  options.length -= truncatedStr.length - options.length
+  return truncate(str, options)
+}
+
+function pageToStartAndCount (page: number, itemsPerPage: number) {
+  const start = (page - 1) * itemsPerPage
+
+  return { start, count: itemsPerPage }
+}
+
+// ---------------------------------------------------------------------------
+
+type SemVersion = { major: number, minor: number, patch: number }
+
+/**
+ * Parses a semantic version string into its separate components.
+ * Fairly lax, and allows for missing or additional segments in the string.
+ *
+ * @param s String to parse semantic version from.
+ * @returns Major, minor, and patch version, or null if string does not follow semantic version conventions.
+ */
+function parseSemVersion (s: string) {
+  const parsed = s.match(/v?(\d+)\.(\d+)(?:\.(\d+))?/i)
+
+  if (!parsed) return null
+
+  return {
+    major: parseInt(parsed[1]),
+    minor: parseInt(parsed[2]),
+    patch: parsed[3] ? parseInt(parsed[3]) : 0
+  } as SemVersion
+}
+
+// ---------------------------------------------------------------------------
+
+function execShell (command: string, options?: ExecOptions) {
+  return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
+    exec(command, options, (err, stdout, stderr) => {
+      // eslint-disable-next-line prefer-promise-reject-errors
+      if (err) return rej({ err, stdout, stderr })
+
+      return res({ stdout, stderr })
+    })
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function generateRSAKeyPairPromise (size: number) {
+  return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
+    const options: RSAKeyPairOptions<'pem', 'pem'> = {
+      modulusLength: size,
+      publicKeyEncoding: {
+        type: 'spki',
+        format: 'pem'
+      },
+      privateKeyEncoding: {
+        type: 'pkcs1',
+        format: 'pem'
+      }
+    }
+
+    generateKeyPair('rsa', options, (err, publicKey, privateKey) => {
+      if (err) return rej(err)
+
+      return res({ publicKey, privateKey })
+    })
+  })
+}
+
+function generateED25519KeyPairPromise () {
+  return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
+    const options: ED25519KeyPairOptions<'pem', 'pem'> = {
+      publicKeyEncoding: {
+        type: 'spki',
+        format: 'pem'
+      },
+      privateKeyEncoding: {
+        type: 'pkcs8',
+        format: 'pem'
+      }
+    }
+
+    generateKeyPair('ed25519', options, (err, publicKey, privateKey) => {
+      if (err) return rej(err)
+
+      return res({ publicKey, privateKey })
+    })
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
+const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
+const execPromise2 = promisify2<string, any, string>(exec)
+const execPromise = promisify1<string, string>(exec)
+const pipelinePromise = promisify(pipeline)
+
+// ---------------------------------------------------------------------------
+
+export {
+  objectConverter,
+  mapToJSON,
+
+  sanitizeUrl,
+  sanitizeHost,
+
+  execShell,
+
+  pageToStartAndCount,
+  peertubeTruncate,
+
+  scryptPromise,
+
+  randomBytesPromise,
+
+  generateRSAKeyPairPromise,
+  generateED25519KeyPairPromise,
+
+  execPromise2,
+  execPromise,
+  pipelinePromise,
+
+  parseSemVersion
+}

server/core/helpers/custom-jsonld-signature.ts

@@ -0,0 +1,81 @@
+import jsonld from 'jsonld'
+
+const STATIC_CACHE = {
+  'https://w3id.org/security/v1': {
+    '@context': {
+      id: '@id',
+      type: '@type',
+
+      dc: 'http://purl.org/dc/terms/',
+      sec: 'https://w3id.org/security#',
+      xsd: 'http://www.w3.org/2001/XMLSchema#',
+
+      EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016',
+      Ed25519Signature2018: 'sec:Ed25519Signature2018',
+      EncryptedMessage: 'sec:EncryptedMessage',
+      GraphSignature2012: 'sec:GraphSignature2012',
+      LinkedDataSignature2015: 'sec:LinkedDataSignature2015',
+      LinkedDataSignature2016: 'sec:LinkedDataSignature2016',
+      CryptographicKey: 'sec:Key',
+
+      authenticationTag: 'sec:authenticationTag',
+      canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm',
+      cipherAlgorithm: 'sec:cipherAlgorithm',
+      cipherData: 'sec:cipherData',
+      cipherKey: 'sec:cipherKey',
+      created: { '@id': 'dc:created', '@type': 'xsd:dateTime' },
+      creator: { '@id': 'dc:creator', '@type': '@id' },
+      digestAlgorithm: 'sec:digestAlgorithm',
+      digestValue: 'sec:digestValue',
+      domain: 'sec:domain',
+      encryptionKey: 'sec:encryptionKey',
+      expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
+      expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
+      initializationVector: 'sec:initializationVector',
+      iterationCount: 'sec:iterationCount',
+      nonce: 'sec:nonce',
+      normalizationAlgorithm: 'sec:normalizationAlgorithm',
+      owner: { '@id': 'sec:owner', '@type': '@id' },
+      password: 'sec:password',
+      privateKey: { '@id': 'sec:privateKey', '@type': '@id' },
+      privateKeyPem: 'sec:privateKeyPem',
+      publicKey: { '@id': 'sec:publicKey', '@type': '@id' },
+      publicKeyBase58: 'sec:publicKeyBase58',
+      publicKeyPem: 'sec:publicKeyPem',
+      publicKeyWif: 'sec:publicKeyWif',
+      publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' },
+      revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' },
+      salt: 'sec:salt',
+      signature: 'sec:signature',
+      signatureAlgorithm: 'sec:signingAlgorithm',
+      signatureValue: 'sec:signatureValue'
+    }
+  }
+}
+
+const localCache = new Map<string, any>()
+
+const nodeDocumentLoader = (jsonld as any).documentLoaders.node();
+
+/* eslint-disable no-import-assign */
+(jsonld as any).documentLoader = async (url: string) => {
+  if (url in STATIC_CACHE) {
+    return {
+      contextUrl: null,
+      document: STATIC_CACHE[url],
+      documentUrl: url
+    }
+  }
+
+  if (localCache.has(url)) return localCache.get(url)
+
+  const remoteDoc = await nodeDocumentLoader(url)
+
+  if (localCache.size < 100) {
+    localCache.set(url, remoteDoc)
+  }
+
+  return remoteDoc
+}
+
+export { jsonld }

server/core/helpers/custom-validators/abuses.ts

@@ -0,0 +1,68 @@
+import validator from 'validator'
+import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils'
+import { AbuseCreate, AbuseFilter, AbusePredefinedReasonsString, AbuseVideoIs } from '@peertube/peertube-models'
+import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists, isArray } from './misc.js'
+
+const ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSES
+const ABUSE_MESSAGES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSE_MESSAGES
+
+function isAbuseReasonValid (value: string) {
+  return exists(value) && validator.default.isLength(value, ABUSES_CONSTRAINTS_FIELDS.REASON)
+}
+
+function isAbusePredefinedReasonValid (value: AbusePredefinedReasonsString) {
+  return exists(value) && value in abusePredefinedReasonsMap
+}
+
+function isAbuseFilterValid (value: AbuseFilter) {
+  return value === 'video' || value === 'comment' || value === 'account'
+}
+
+function areAbusePredefinedReasonsValid (value: AbusePredefinedReasonsString[]) {
+  return exists(value) && isArray(value) && value.every(v => v in abusePredefinedReasonsMap)
+}
+
+function isAbuseTimestampValid (value: number) {
+  return value === null || (exists(value) && validator.default.isInt('' + value, { min: 0 }))
+}
+
+function isAbuseTimestampCoherent (endAt: number, { req }) {
+  const startAt = (req.body as AbuseCreate).video.startAt
+
+  return exists(startAt) && endAt > startAt
+}
+
+function isAbuseModerationCommentValid (value: string) {
+  return exists(value) && validator.default.isLength(value, ABUSES_CONSTRAINTS_FIELDS.MODERATION_COMMENT)
+}
+
+function isAbuseStateValid (value: string) {
+  return exists(value) && ABUSE_STATES[value] !== undefined
+}
+
+function isAbuseVideoIsValid (value: AbuseVideoIs) {
+  return exists(value) && (
+    value === 'deleted' ||
+    value === 'blacklisted'
+  )
+}
+
+function isAbuseMessageValid (value: string) {
+  return exists(value) && validator.default.isLength(value, ABUSE_MESSAGES_CONSTRAINTS_FIELDS.MESSAGE)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isAbuseReasonValid,
+  isAbuseFilterValid,
+  isAbusePredefinedReasonValid,
+  isAbuseMessageValid,
+  areAbusePredefinedReasonsValid,
+  isAbuseTimestampValid,
+  isAbuseTimestampCoherent,
+  isAbuseModerationCommentValid,
+  isAbuseStateValid,
+  isAbuseVideoIsValid
+}

server/core/helpers/custom-validators/accounts.ts

@@ -0,0 +1,22 @@
+import { isUserDescriptionValid, isUserUsernameValid } from './users.js'
+import { exists } from './misc.js'
+
+function isAccountNameValid (value: string) {
+  return isUserUsernameValid(value)
+}
+
+function isAccountIdValid (value: string) {
+  return exists(value)
+}
+
+function isAccountDescriptionValid (value: string) {
+  return isUserDescriptionValid(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isAccountIdValid,
+  isAccountDescriptionValid,
+  isAccountNameValid
+}

server/core/helpers/custom-validators/activitypub/activity.ts

@@ -0,0 +1,148 @@
+import validator from 'validator'
+import { Activity, ActivityType } from '@peertube/peertube-models'
+import { isAbuseReasonValid } from '../abuses.js'
+import { exists } from '../misc.js'
+import { sanitizeAndCheckActorObject } from './actor.js'
+import { isCacheFileObjectValid } from './cache-file.js'
+import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc.js'
+import { isPlaylistObjectValid } from './playlist.js'
+import { sanitizeAndCheckVideoCommentObject } from './video-comments.js'
+import { sanitizeAndCheckVideoTorrentObject } from './videos.js'
+import { isWatchActionObjectValid } from './watch-action.js'
+
+export function isRootActivityValid (activity: any) {
+  return isCollection(activity) || isActivity(activity)
+}
+
+function isCollection (activity: any) {
+  return (activity.type === 'Collection' || activity.type === 'OrderedCollection') &&
+    validator.default.isInt(activity.totalItems, { min: 0 }) &&
+    Array.isArray(activity.items)
+}
+
+function isActivity (activity: any) {
+  return isActivityPubUrlValid(activity.id) &&
+    exists(activity.actor) &&
+    (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id))
+}
+
+// ---------------------------------------------------------------------------
+
+const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = {
+  Create: isCreateActivityValid,
+  Update: isUpdateActivityValid,
+  Delete: isDeleteActivityValid,
+  Follow: isFollowActivityValid,
+  Accept: isAcceptActivityValid,
+  Reject: isRejectActivityValid,
+  Announce: isAnnounceActivityValid,
+  Undo: isUndoActivityValid,
+  Like: isLikeActivityValid,
+  View: isViewActivityValid,
+  Flag: isFlagActivityValid,
+  Dislike: isDislikeActivityValid,
+  ApproveReply: isApproveReplyActivityValid,
+  RejectReply: isRejectReplyActivityValid
+}
+
+export function isActivityValid (activity: any) {
+  const checker = activityCheckers[activity.type]
+  // Unknown activity type
+  if (!checker) return false
+
+  return checker(activity)
+}
+
+export function isFlagActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Flag') &&
+    isAbuseReasonValid(activity.content) &&
+    isActivityPubUrlValid(activity.object)
+}
+
+export function isLikeActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Like') &&
+    isObjectValid(activity.object)
+}
+
+export function isDislikeActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Dislike') &&
+    isObjectValid(activity.object)
+}
+
+export function isAnnounceActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Announce') &&
+    isObjectValid(activity.object)
+}
+
+export function isViewActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'View') &&
+    isActivityPubUrlValid(activity.actor) &&
+    isActivityPubUrlValid(activity.object)
+}
+
+export function isCreateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Create') &&
+    (
+      isViewActivityValid(activity.object) ||
+      isDislikeActivityValid(activity.object) ||
+      isFlagActivityValid(activity.object) ||
+      isPlaylistObjectValid(activity.object) ||
+      isWatchActionObjectValid(activity.object) ||
+
+      isCacheFileObjectValid(activity.object) ||
+      sanitizeAndCheckVideoCommentObject(activity.object) ||
+      sanitizeAndCheckVideoTorrentObject(activity.object)
+    )
+}
+
+export function isUpdateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Update') &&
+    (
+      isCacheFileObjectValid(activity.object) ||
+      isPlaylistObjectValid(activity.object) ||
+      sanitizeAndCheckVideoTorrentObject(activity.object) ||
+      sanitizeAndCheckActorObject(activity.object)
+    )
+}
+
+export function isDeleteActivityValid (activity: any) {
+  // We don't really check objects
+  return isBaseActivityValid(activity, 'Delete') &&
+    isObjectValid(activity.object)
+}
+
+export function isFollowActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Follow') &&
+    isObjectValid(activity.object)
+}
+
+export function isAcceptActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Accept')
+}
+
+export function isRejectActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Reject')
+}
+
+export function isUndoActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Undo') &&
+    (
+      isFollowActivityValid(activity.object) ||
+      isLikeActivityValid(activity.object) ||
+      isDislikeActivityValid(activity.object) ||
+      isAnnounceActivityValid(activity.object) ||
+      isCreateActivityValid(activity.object)
+    )
+}
+
+export function isApproveReplyActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'ApproveReply') &&
+    isActivityPubUrlValid(activity.object) &&
+    isActivityPubUrlValid(activity.inReplyTo)
+}
+
+export function isRejectReplyActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'RejectReply') &&
+    isActivityPubUrlValid(activity.object) &&
+    isActivityPubUrlValid(activity.inReplyTo)
+}

server/core/helpers/custom-validators/activitypub/actor.ts

@@ -0,0 +1,143 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js'
+import { exists, isArray, isDateValid } from '../misc.js'
+import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc.js'
+import { isHostValid } from '../servers.js'
+import { peertubeTruncate } from '@server/helpers/core-utils.js'
+
+function isActorEndpointsObjectValid (endpointObject: any) {
+  if (endpointObject?.sharedInbox) {
+    return isActivityPubUrlValid(endpointObject.sharedInbox)
+  }
+
+  // Shared inbox is optional
+  return true
+}
+
+function isActorPublicKeyObjectValid (publicKeyObject: any) {
+  return isActivityPubUrlValid(publicKeyObject.id) &&
+    isActivityPubUrlValid(publicKeyObject.owner) &&
+    isActorPublicKeyValid(publicKeyObject.publicKeyPem)
+}
+
+const actorTypes = new Set([ 'Person', 'Application', 'Group', 'Service', 'Organization' ])
+function isActorTypeValid (type: string) {
+  return actorTypes.has(type)
+}
+
+function isActorPublicKeyValid (publicKey: string) {
+  return exists(publicKey) &&
+    typeof publicKey === 'string' &&
+    publicKey.startsWith('-----BEGIN PUBLIC KEY-----') &&
+    publicKey.includes('-----END PUBLIC KEY-----') &&
+    validator.default.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY)
+}
+
+const actorNameAlphabet = '[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\\-_.:]'
+const actorNameRegExp = new RegExp(`^${actorNameAlphabet}+$`)
+function isActorPreferredUsernameValid (preferredUsername: string) {
+  return exists(preferredUsername) && validator.default.matches(preferredUsername, actorNameRegExp)
+}
+
+function isActorPrivateKeyValid (privateKey: string) {
+  return exists(privateKey) &&
+    typeof privateKey === 'string' &&
+    (privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') || privateKey.startsWith('-----BEGIN PRIVATE KEY-----')) &&
+    // Sometimes there is a \n at the end, so just assert the string contains the end mark
+    (privateKey.includes('-----END RSA PRIVATE KEY-----') || privateKey.includes('-----END PRIVATE KEY-----')) &&
+    validator.default.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY)
+}
+
+function isActorFollowingCountValid (value: string) {
+  return exists(value) && validator.default.isInt('' + value, { min: 0 })
+}
+
+function isActorFollowersCountValid (value: string) {
+  return exists(value) && validator.default.isInt('' + value, { min: 0 })
+}
+
+function isActorDeleteActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Delete')
+}
+
+function sanitizeAndCheckActorObject (actor: any) {
+  if (!isActorTypeValid(actor.type)) return false
+
+  normalizeActor(actor)
+
+  return exists(actor) &&
+    isActivityPubUrlValid(actor.id) &&
+    isActivityPubUrlValid(actor.inbox) &&
+    isActorPreferredUsernameValid(actor.preferredUsername) &&
+    isActivityPubUrlValid(actor.url) &&
+    isActorPublicKeyObjectValid(actor.publicKey) &&
+    isActorEndpointsObjectValid(actor.endpoints) &&
+
+    (!actor.outbox || isActivityPubUrlValid(actor.outbox)) &&
+    (!actor.following || isActivityPubUrlValid(actor.following)) &&
+    (!actor.followers || isActivityPubUrlValid(actor.followers)) &&
+
+    setValidAttributedTo(actor) &&
+    setValidDescription(actor) &&
+    // If this is a group (a channel), it should be attributed to an account
+    // In PeerTube we use this to attach a video channel to a specific account
+    (actor.type !== 'Group' || actor.attributedTo.length !== 0)
+}
+
+function normalizeActor (actor: any) {
+  if (!actor) return
+
+  if (!actor.url) {
+    actor.url = actor.id
+  } else if (typeof actor.url !== 'string') {
+    actor.url = actor.url.href || actor.url.url
+  }
+
+  if (!isDateValid(actor.published)) actor.published = undefined
+
+  if (actor.summary && typeof actor.summary === 'string') {
+    actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max })
+
+    if (actor.summary.length < CONSTRAINTS_FIELDS.USERS.DESCRIPTION.min) {
+      actor.summary = null
+    }
+  }
+}
+
+function isValidActorHandle (handle: string) {
+  if (!exists(handle)) return false
+
+  const parts = handle.split('@')
+  if (parts.length !== 2) return false
+
+  return isHostValid(parts[1])
+}
+
+function areValidActorHandles (handles: string[]) {
+  return isArray(handles) && handles.every(h => isValidActorHandle(h))
+}
+
+function setValidDescription (obj: any) {
+  if (!obj.summary) obj.summary = null
+
+  return true
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  normalizeActor,
+  actorNameAlphabet,
+  areValidActorHandles,
+  isActorEndpointsObjectValid,
+  isActorPublicKeyObjectValid,
+  isActorTypeValid,
+  isActorPublicKeyValid,
+  isActorPreferredUsernameValid,
+  isActorPrivateKeyValid,
+  isActorFollowingCountValid,
+  isActorFollowersCountValid,
+  isActorDeleteActivityValid,
+  sanitizeAndCheckActorObject,
+  isValidActorHandle
+}

server/core/helpers/custom-validators/activitypub/cache-file.ts

@@ -0,0 +1,33 @@
+import { CacheFileObject } from '@peertube/peertube-models'
+import { MIMETYPES } from '@server/initializers/constants.js'
+import validator from 'validator'
+import { isDateValid } from '../misc.js'
+import { isActivityPubUrlValid } from './misc.js'
+
+export function isCacheFileObjectValid (object: CacheFileObject) {
+  if (!object || object.type !== 'CacheFile') return false
+
+  return (!object.expires || isDateValid(object.expires)) &&
+    isActivityPubUrlValid(object.object) &&
+    (isRedundancyUrlVideoValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
+}
+
+// ---------------------------------------------------------------------------
+
+function isPlaylistRedundancyUrlValid (url: any) {
+  return url.type === 'Link' &&
+    (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
+    isActivityPubUrlValid(url.href)
+}
+
+// TODO: compat with < 6.1, use isRemoteVideoUrlValid instead in 7.0
+function isRedundancyUrlVideoValid (url: any) {
+  const size = url.size || url['_:size']
+  const fps = url.fps || url['_fps']
+
+  return MIMETYPES.AP_VIDEO.MIMETYPE_EXT[url.mediaType] &&
+    isActivityPubUrlValid(url.href) &&
+    validator.default.isInt(url.height + '', { min: 0 }) &&
+    validator.default.isInt(size + '', { min: 0 }) &&
+    (!fps || validator.default.isInt(fps + '', { min: -1 }))
+}

server/core/helpers/custom-validators/activitypub/misc.ts

@@ -0,0 +1,76 @@
+import validator from 'validator'
+import { CONFIG } from '@server/initializers/config.js'
+import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js'
+import { exists } from '../misc.js'
+
+function isUrlValid (url: string) {
+  const isURLOptions = {
+    require_host: true,
+    require_tld: true,
+    require_protocol: true,
+    require_valid_protocol: true,
+    protocols: [ 'http', 'https' ]
+  }
+
+  // We validate 'localhost', so we don't have the top level domain
+  if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') {
+    isURLOptions.require_tld = false
+  }
+
+  return exists(url) && validator.default.isURL('' + url, isURLOptions)
+}
+
+function isActivityPubUrlValid (url: string) {
+  return isUrlValid(url) && validator.default.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
+}
+
+function isBaseActivityValid (activity: any, type: string) {
+  return activity.type === type &&
+    isActivityPubUrlValid(activity.id) &&
+    isObjectValid(activity.actor) &&
+    isUrlCollectionValid(activity.to) &&
+    isUrlCollectionValid(activity.cc)
+}
+
+function isUrlCollectionValid (collection: any) {
+  return collection === undefined ||
+    (Array.isArray(collection) && collection.every(t => isActivityPubUrlValid(t)))
+}
+
+function isObjectValid (object: any) {
+  return exists(object) &&
+    (
+      isActivityPubUrlValid(object) || isActivityPubUrlValid(object.id)
+    )
+}
+
+function setValidAttributedTo (obj: any) {
+  if (Array.isArray(obj.attributedTo) === false) {
+    obj.attributedTo = []
+    return true
+  }
+
+  obj.attributedTo = obj.attributedTo.filter(a => {
+    return isActivityPubUrlValid(a) ||
+      ((a.type === 'Group' || a.type === 'Person') && isActivityPubUrlValid(a.id))
+  })
+
+  return true
+}
+
+function isActivityPubVideoDurationValid (value: string) {
+  // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
+  return exists(value) &&
+    typeof value === 'string' &&
+    value.startsWith('PT') &&
+    value.endsWith('S')
+}
+
+export {
+  isUrlValid,
+  isActivityPubUrlValid,
+  isBaseActivityValid,
+  setValidAttributedTo,
+  isObjectValid,
+  isActivityPubVideoDurationValid
+}

server/core/helpers/custom-validators/activitypub/playlist.ts

@@ -0,0 +1,25 @@
+import { PlaylistElementObject, PlaylistObject } from '@peertube/peertube-models'
+import validator from 'validator'
+import { exists, isDateValid, isUUIDValid } from '../misc.js'
+import { isVideoPlaylistNameValid } from '../video-playlists.js'
+import { isActivityPubUrlValid } from './misc.js'
+
+export function isPlaylistObjectValid (object: PlaylistObject) {
+  if (!object || object.type !== 'Playlist') return false
+
+  // TODO: compat with < 6.1, remove in 7.0
+  if (!object.uuid && object['identifier']) object.uuid = object['identifier']
+
+  return validator.default.isInt(object.totalItems + '') &&
+    isVideoPlaylistNameValid(object.name) &&
+    isUUIDValid(object.uuid) &&
+    isDateValid(object.published) &&
+    isDateValid(object.updated)
+}
+
+export function isPlaylistElementObjectValid (object: PlaylistElementObject) {
+  return exists(object) &&
+    object.type === 'PlaylistElement' &&
+    validator.default.isInt(object.position + '') &&
+    isActivityPubUrlValid(object.url)
+}

server/core/helpers/custom-validators/activitypub/signature.ts

@@ -0,0 +1,22 @@
+import { exists } from '../misc.js'
+import { isActivityPubUrlValid } from './misc.js'
+
+function isSignatureTypeValid (signatureType: string) {
+  return exists(signatureType) && signatureType === 'RsaSignature2017'
+}
+
+function isSignatureCreatorValid (signatureCreator: string) {
+  return exists(signatureCreator) && isActivityPubUrlValid(signatureCreator)
+}
+
+function isSignatureValueValid (signatureValue: string) {
+  return exists(signatureValue) && signatureValue.length > 0
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isSignatureTypeValid,
+  isSignatureCreatorValid,
+  isSignatureValueValid
+}

server/core/helpers/custom-validators/activitypub/video-chapters.ts

@@ -0,0 +1,15 @@
+import { isArray } from '../misc.js'
+import { isVideoChapterTitleValid, isVideoChapterTimecodeValid } from '../video-chapters.js'
+import { isActivityPubUrlValid } from './misc.js'
+import { VideoChaptersObject } from '@peertube/peertube-models'
+
+export function isVideoChaptersObjectValid (object: VideoChaptersObject) {
+  if (!object) return false
+  if (!isActivityPubUrlValid(object.id)) return false
+
+  if (!isArray(object.hasPart)) return false
+
+  return object.hasPart.every(part => {
+    return isVideoChapterTitleValid(part.name) && isVideoChapterTimecodeValid(part.startOffset)
+  })
+}

server/core/helpers/custom-validators/activitypub/video-comments.ts

@@ -0,0 +1,58 @@
+import { hasAPPublic } from '@server/helpers/activity-pub-utils.js'
+import validator from 'validator'
+import { exists, isArray, isDateValid } from '../misc.js'
+import { isActivityPubUrlValid } from './misc.js'
+import { ActivityTombstoneObject, VideoCommentObject } from '@peertube/peertube-models'
+
+function sanitizeAndCheckVideoCommentObject (comment: VideoCommentObject | ActivityTombstoneObject) {
+  if (!comment) return false
+
+  if (!isCommentTypeValid(comment)) return false
+
+  normalizeComment(comment)
+
+  if (comment.type === 'Tombstone') {
+    return isActivityPubUrlValid(comment.id) &&
+      isDateValid(comment.published) &&
+      isDateValid(comment.deleted) &&
+      isActivityPubUrlValid(comment.url)
+  }
+
+  return isActivityPubUrlValid(comment.id) &&
+    isCommentContentValid(comment.content) &&
+    isActivityPubUrlValid(comment.inReplyTo) &&
+    isDateValid(comment.published) &&
+    isActivityPubUrlValid(comment.url) &&
+    isArray(comment.to) &&
+    (!exists(comment.replyApproval) || isActivityPubUrlValid(comment.replyApproval)) &&
+    (hasAPPublic(comment.to) || hasAPPublic(comment.cc)) // Only accept public comments
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  sanitizeAndCheckVideoCommentObject
+}
+
+// ---------------------------------------------------------------------------
+
+function isCommentContentValid (content: any) {
+  return exists(content) && validator.default.isLength('' + content, { min: 1 })
+}
+
+function normalizeComment (comment: any) {
+  if (!comment) return
+
+  if (typeof comment.url !== 'string') {
+    if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url
+    else comment.url = comment.id
+  }
+}
+
+function isCommentTypeValid (comment: any): boolean {
+  if (comment.type === 'Note') return true
+
+  if (comment.type === 'Tombstone' && comment.formerType === 'Note') return true
+
+  return false
+}

server/core/helpers/custom-validators/activitypub/videos.ts

@@ -0,0 +1,255 @@
+import {
+  ActivityPubStoryboard,
+  ActivityTrackerUrlObject,
+  ActivityVideoFileMetadataUrlObject,
+  LiveVideoLatencyMode,
+  VideoCommentPolicy,
+  VideoObject,
+  VideoState
+} from '@peertube/peertube-models'
+import { logger } from '@server/helpers/logger.js'
+import { spdxToPeertubeLicence } from '@server/helpers/video.js'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'
+import { peertubeTruncate } from '../../core-utils.js'
+import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js'
+import { isLiveLatencyModeValid } from '../video-lives.js'
+import {
+  isVideoCommentsPolicyValid,
+  isVideoDescriptionValid,
+  isVideoDurationValid,
+  isVideoNameValid,
+  isVideoStateValid,
+  isVideoTagValid,
+  isVideoViewsValid
+} from '../videos.js'
+import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc.js'
+
+export function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
+  return isBaseActivityValid(activity, 'Update') &&
+    sanitizeAndCheckVideoTorrentObject(activity.object)
+}
+
+export function sanitizeAndCheckVideoTorrentObject (video: VideoObject) {
+  if (!video || video.type !== 'Video') return false
+
+  const fail = (field: string) => {
+    logger.debug(`Video field is not valid to PeerTube: ${field}`, { video })
+    return false
+  }
+
+  if (!setValidRemoteTags(video)) return fail('tags')
+  if (!setValidRemoteVideoUrls(video)) return fail('urls')
+  if (!setRemoteVideoContent(video)) return fail('content')
+  if (!setValidAttributedTo(video)) return fail('attributedTo')
+  if (!setValidRemoteCaptions(video)) return fail('captions')
+  if (!setValidRemoteIcon(video)) return fail('icons')
+  if (!setValidStoryboard(video)) return fail('preview (storyboard)')
+  if (!setValidLicence(video)) return fail('licence')
+
+  // TODO: compat with < 6.1, remove in 7.0
+  if (!video.uuid && video['identifier']) video.uuid = video['identifier']
+
+  // Default attributes
+  if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
+  if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false
+  if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true
+  if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false
+  if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false
+  if (!isBooleanValid(video.permanentLive)) video.permanentLive = false
+  if (!isBooleanValid(video.sensitive)) video.sensitive = false
+  if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT
+
+  if (video.commentsPolicy) {
+    if (!isVideoCommentsPolicyValid(video.commentsPolicy)) {
+      video.commentsPolicy = VideoCommentPolicy.DISABLED
+    }
+  } else if (video.commentsEnabled === true) { // Fallback to deprecated attribute
+    video.commentsPolicy = VideoCommentPolicy.ENABLED
+  } else {
+    video.commentsPolicy = VideoCommentPolicy.DISABLED
+  }
+
+  if (!isActivityPubUrlValid(video.id)) return fail('id')
+  if (!isVideoNameValid(video.name)) return fail('name')
+
+  if (!isActivityPubVideoDurationValid(video.duration)) return fail('duration format')
+  if (!isVideoDurationValid(video.duration.replace(/[^0-9]+/g, ''))) return fail('duration')
+
+  if (!isUUIDValid(video.uuid)) return fail('uuid')
+
+  if (exists(video.category) && !isRemoteNumberIdentifierValid(video.category)) return fail('category')
+  if (exists(video.language) && !isRemoteStringIdentifierValid(video.language)) return fail('language')
+
+  if (!isVideoViewsValid(video.views)) return fail('views')
+  if (!isDateValid(video.published)) return fail('published')
+  if (!isDateValid(video.updated)) return fail('updated')
+
+  if (exists(video.originallyPublishedAt) && !isDateValid(video.originallyPublishedAt)) return fail('originallyPublishedAt')
+  if (exists(video.uploadDate) && !isDateValid(video.uploadDate)) return fail('uploadDate')
+  if (exists(video.content) && !isRemoteVideoContentValid(video.mediaType, video.content)) return fail('mediaType/content')
+
+  if (video.attributedTo.length === 0) return fail('attributedTo')
+
+  return true
+}
+
+export function isRemoteVideoUrlValid (url: any) {
+  return url.type === 'Link' &&
+    // Video file link
+    (
+      MIMETYPES.AP_VIDEO.MIMETYPE_EXT[url.mediaType] &&
+      isActivityPubUrlValid(url.href) &&
+      validator.default.isInt(url.height + '', { min: 0 }) &&
+      validator.default.isInt(url.size + '', { min: 0 }) &&
+      (!url.fps || validator.default.isInt(url.fps + '', { min: -1 }))
+    ) ||
+    // Torrent link
+    (
+      MIMETYPES.AP_TORRENT.MIMETYPE_EXT[url.mediaType] &&
+      isActivityPubUrlValid(url.href) &&
+      validator.default.isInt(url.height + '', { min: 0 })
+    ) ||
+    // Magnet link
+    (
+      MIMETYPES.AP_MAGNET.MIMETYPE_EXT[url.mediaType] &&
+      validator.default.isLength(url.href, { min: 5 }) &&
+      validator.default.isInt(url.height + '', { min: 0 })
+    ) ||
+    // HLS playlist link
+    (
+      (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
+      isActivityPubUrlValid(url.href) &&
+      isArray(url.tag)
+    ) ||
+    isAPVideoTrackerUrlObject(url) ||
+    isAPVideoFileUrlMetadataObject(url)
+}
+
+export function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject {
+  return url &&
+    url.type === 'Link' &&
+    url.mediaType === 'application/json' &&
+    isArray(url.rel) && url.rel.includes('metadata')
+}
+
+export function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject {
+  return isArray(url.rel) &&
+    url.rel.includes('tracker') &&
+    isActivityPubUrlValid(url.href)
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function setValidRemoteTags (video: VideoObject) {
+  if (Array.isArray(video.tag) === false) video.tag = []
+
+  video.tag = video.tag.filter(t => t.type === 'Hashtag' && isVideoTagValid(t.name))
+
+  return true
+}
+
+function setValidRemoteCaptions (video: VideoObject) {
+  if (!video.subtitleLanguage) video.subtitleLanguage = []
+
+  if (Array.isArray(video.subtitleLanguage) === false) return false
+
+  video.subtitleLanguage = video.subtitleLanguage.filter(caption => {
+    if (!isActivityPubUrlValid(caption.url)) caption.url = null
+
+    return isRemoteStringIdentifierValid(caption)
+  })
+
+  return true
+}
+
+function isRemoteNumberIdentifierValid (data: any) {
+  return validator.default.isInt(data.identifier, { min: 0 })
+}
+
+function isRemoteStringIdentifierValid (data: any) {
+  return typeof data.identifier === 'string'
+}
+
+function isRemoteVideoContentValid (mediaType: string, content: string) {
+  return (mediaType === 'text/markdown' || mediaType === 'text/html') && isVideoDescriptionValid(content)
+}
+
+function setValidRemoteIcon (video: any) {
+  if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ]
+  if (!video.icon) video.icon = []
+
+  video.icon = video.icon.filter(icon => {
+    return icon.type === 'Image' &&
+      isActivityPubUrlValid(icon.url) &&
+      icon.mediaType === 'image/jpeg' &&
+      validator.default.isInt(icon.width + '', { min: 0 }) &&
+      validator.default.isInt(icon.height + '', { min: 0 })
+  })
+
+  return video.icon.length !== 0
+}
+
+function setValidRemoteVideoUrls (video: any) {
+  if (Array.isArray(video.url) === false) return false
+
+  video.url = video.url.filter(u => isRemoteVideoUrlValid(u))
+
+  return true
+}
+
+function setRemoteVideoContent (video: VideoObject) {
+  if (video.content) {
+    video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max })
+  }
+
+  return true
+}
+
+function setValidLicence (video: VideoObject) {
+  if (!exists(video.licence)) return true
+
+  if (validator.default.isInt(video.licence.identifier)) return isRemoteNumberIdentifierValid(video.licence)
+
+  const spdx = spdxToPeertubeLicence(video.licence.identifier)
+  video.licence.identifier = spdx
+    ? spdx + ''
+    : undefined
+
+  return true
+}
+
+function setValidStoryboard (video: VideoObject) {
+  if (!video.preview) return true
+  if (!Array.isArray(video.preview)) return false
+
+  video.preview = video.preview.filter(p => isStorybordValid(p))
+
+  return true
+}
+
+function isStorybordValid (preview: ActivityPubStoryboard) {
+  if (!preview) return false
+
+  if (
+    preview.type !== 'Image' ||
+    !isArray(preview.rel) ||
+    !preview.rel.includes('storyboard')
+  ) {
+    return false
+  }
+
+  preview.url = preview.url.filter(u => {
+    return u.mediaType === 'image/jpeg' &&
+      isActivityPubUrlValid(u.href) &&
+      validator.default.isInt(u.width + '', { min: 0 }) &&
+      validator.default.isInt(u.height + '', { min: 0 }) &&
+      validator.default.isInt(u.tileWidth + '', { min: 0 }) &&
+      validator.default.isInt(u.tileHeight + '', { min: 0 }) &&
+      isActivityPubVideoDurationValid(u.tileDuration)
+  })
+
+  return preview.url.length !== 0
+}

server/core/helpers/custom-validators/activitypub/watch-action.ts

@@ -0,0 +1,51 @@
+import { arrayify } from '@peertube/peertube-core-utils'
+import { WatchActionObject } from '@peertube/peertube-models'
+import { isDateValid, isUUIDValid } from '../misc.js'
+import { isVideoTimeValid } from '../video-view.js'
+import { isActivityPubVideoDurationValid, isObjectValid } from './misc.js'
+
+function isWatchActionObjectValid (action: WatchActionObject) {
+  if (!action || action.type !== 'WatchAction') return false
+
+  // TODO: compat with < 6.1, remove in 7.0
+  if (!action.uuid && action['identifier']) action.uuid = action['identifier']
+
+  if (action['_:actionStatus'] && !action.actionStatus) action.actionStatus = action['_:actionStatus']
+  if (action['_:watchSections'] && !action.watchSections) action.watchSections = arrayify(action['_:watchSections'])
+
+  return isObjectValid(action.id) &&
+    isActivityPubVideoDurationValid(action.duration) &&
+    isDateValid(action.startTime) &&
+    isDateValid(action.endTime) &&
+    isLocationValid(action.location) &&
+    isUUIDValid(action.uuid) &&
+    isObjectValid(action.object) &&
+    areWatchSectionsValid(action.watchSections)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isWatchActionObjectValid
+}
+
+// ---------------------------------------------------------------------------
+
+function isLocationValid (location: any) {
+  if (!location) return true
+  if (typeof location !== 'object') return false
+
+  if (location.addressCountry && typeof location.addressCountry !== 'string') return false
+  if (location.addressRegion && typeof location.addressRegion !== 'string') return false
+
+  return true
+}
+
+function areWatchSectionsValid (sections: WatchActionObject['watchSections']) {
+  return Array.isArray(sections) && sections.every(s => {
+    // TODO: compat with < 6.1, remove in 7.0
+    if (s['_:endTimestamp'] && !s.endTimestamp) s.endTimestamp = s['_:endTimestamp']
+
+    return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp)
+  })
+}

server/core/helpers/custom-validators/actor-images.ts

@@ -0,0 +1,23 @@
+import { UploadFilesForCheck } from 'express'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { isFileValid } from './misc.js'
+
+const imageMimeTypes = CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME
+  .map(v => v.replace('.', ''))
+  .join('|')
+const imageMimeTypesRegex = `image/(${imageMimeTypes})`
+
+function isActorImageFile (files: UploadFilesForCheck, fieldname: string) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: imageMimeTypesRegex,
+    field: fieldname,
+    maxSize: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isActorImageFile
+}

server/core/helpers/custom-validators/bulk.ts

@@ -0,0 +1,9 @@
+function isBulkRemoveCommentsOfScopeValid (value: string) {
+  return value === 'my-videos' || value === 'instance'
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isBulkRemoveCommentsOfScopeValid
+}

server/core/helpers/custom-validators/feeds.ts

@@ -0,0 +1,23 @@
+import { exists } from './misc.js'
+
+function isValidRSSFeed (value: string) {
+  if (!exists(value)) return false
+
+  const feedExtensions = [
+    'xml',
+    'json',
+    'json1',
+    'rss',
+    'rss2',
+    'atom',
+    'atom1'
+  ]
+
+  return feedExtensions.includes(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidRSSFeed
+}

server/core/helpers/custom-validators/follows.ts

@@ -0,0 +1,30 @@
+import { exists, isArray } from './misc.js'
+import { FollowState } from '@peertube/peertube-models'
+
+function isFollowStateValid (value: FollowState) {
+  if (!exists(value)) return false
+
+  return value === 'pending' || value === 'accepted' || value === 'rejected'
+}
+
+function isRemoteHandleValid (value: string) {
+  if (!exists(value)) return false
+  if (typeof value !== 'string') return false
+
+  return value.includes('@')
+}
+
+function isEachUniqueHandleValid (handles: string[]) {
+  return isArray(handles) &&
+    handles.every(handle => {
+      return isRemoteHandleValid(handle) && handles.indexOf(handle) === handles.lastIndexOf(handle)
+    })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isFollowStateValid,
+  isRemoteHandleValid,
+  isEachUniqueHandleValid
+}

server/core/helpers/custom-validators/jobs.ts

@@ -0,0 +1,21 @@
+import { JobState } from '@peertube/peertube-models'
+import { jobTypes } from '@server/lib/job-queue/job-queue.js'
+import { exists } from './misc.js'
+
+const jobStates = new Set<JobState>([ 'active', 'completed', 'failed', 'waiting', 'delayed', 'paused', 'waiting-children', 'prioritized' ])
+
+function isValidJobState (value: JobState) {
+  return exists(value) && jobStates.has(value)
+}
+
+function isValidJobType (value: any) {
+  return exists(value) && jobTypes.includes(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  jobStates,
+  isValidJobState,
+  isValidJobType
+}

server/core/helpers/custom-validators/logs.ts

@@ -0,0 +1,42 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import { ClientLogLevel, ServerLogLevel } from '@peertube/peertube-models'
+import { exists } from './misc.js'
+
+const serverLogLevels = new Set<ServerLogLevel>([ 'debug', 'info', 'warn', 'error' ])
+const clientLogLevels = new Set<ClientLogLevel>([ 'warn', 'error' ])
+
+function isValidLogLevel (value: any) {
+  return exists(value) && serverLogLevels.has(value)
+}
+
+function isValidClientLogMessage (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_MESSAGE)
+}
+
+function isValidClientLogLevel (value: any) {
+  return exists(value) && clientLogLevels.has(value)
+}
+
+function isValidClientLogStackTrace (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_STACK_TRACE)
+}
+
+function isValidClientLogMeta (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_META)
+}
+
+function isValidClientLogUserAgent (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_USER_AGENT)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidLogLevel,
+  isValidClientLogMessage,
+  isValidClientLogStackTrace,
+  isValidClientLogMeta,
+  isValidClientLogLevel,
+  isValidClientLogUserAgent
+}

server/core/helpers/custom-validators/metrics.ts

@@ -0,0 +1,10 @@
+function isValidPlayerMode (value: any) {
+  // TODO: remove webtorrent in v7
+  return value === 'webtorrent' || value === 'web-video' || value === 'p2p-media-loader'
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidPlayerMode
+}

server/core/helpers/custom-validators/misc.ts

@@ -0,0 +1,190 @@
+import 'multer'
+import { UploadFilesForCheck } from 'express'
+import { sep } from 'path'
+import validator from 'validator'
+import { isShortUUID, shortToUUID } from '@peertube/peertube-node-utils'
+
+function exists (value: any) {
+  return value !== undefined && value !== null
+}
+
+function isSafePath (p: string) {
+  return exists(p) &&
+    (p + '').split(sep).every(part => {
+      return [ '..' ].includes(part) === false
+    })
+}
+
+function isSafeFilename (filename: string, extension?: string) {
+  const regex = extension
+    ? new RegExp(`^[a-z0-9-]+\\.${extension}$`)
+    : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`)
+
+  return typeof filename === 'string' && !!filename.match(regex)
+}
+
+function isSafePeerTubeFilenameWithoutExtension (filename: string) {
+  return filename.match(/^[a-z0-9-]+$/)
+}
+
+function isArray (value: any): value is any[] {
+  return Array.isArray(value)
+}
+
+function isNotEmptyIntArray (value: any) {
+  return Array.isArray(value) && value.every(v => validator.default.isInt('' + v)) && value.length !== 0
+}
+
+function isNotEmptyStringArray (value: any) {
+  return Array.isArray(value) && value.every(v => typeof v === 'string' && v.length !== 0) && value.length !== 0
+}
+
+function isArrayOf (value: any, validator: (value: any) => boolean) {
+  return isArray(value) && value.every(v => validator(v))
+}
+
+function isDateValid (value: string) {
+  return exists(value) && validator.default.isISO8601(value)
+}
+
+function isIdValid (value: string) {
+  return exists(value) && validator.default.isInt('' + value)
+}
+
+function isUUIDValid (value: string) {
+  return exists(value) && validator.default.isUUID('' + value, 4)
+}
+
+function areUUIDsValid (values: string[]) {
+  return isArray(values) && values.every(v => isUUIDValid(v))
+}
+
+function isIdOrUUIDValid (value: string) {
+  return isIdValid(value) || isUUIDValid(value)
+}
+
+function isBooleanValid (value: any) {
+  return typeof value === 'boolean' || (typeof value === 'string' && validator.default.isBoolean(value))
+}
+
+function isIntOrNull (value: any) {
+  return value === null || validator.default.isInt('' + value)
+}
+
+// ---------------------------------------------------------------------------
+
+function isFileValid (options: {
+  files: UploadFilesForCheck
+
+  maxSize: number | null
+  mimeTypeRegex: string | null
+
+  field?: string
+
+  optional?: boolean // Default false
+}) {
+  const { files, mimeTypeRegex, field, maxSize, optional = false } = options
+
+  // Should have files
+  if (!files) return optional
+
+  const fileArray = isArray(files)
+    ? files
+    : files[field]
+
+  if (!fileArray || !isArray(fileArray) || fileArray.length === 0) {
+    return optional
+  }
+
+  // The file exists
+  const file = fileArray[0]
+  if (!file?.originalname) return false
+
+  // Check size
+  if ((maxSize !== null) && file.size > maxSize) return false
+
+  if (mimeTypeRegex === null) return true
+
+  return checkMimetypeRegex(file.mimetype, mimeTypeRegex)
+}
+
+function checkMimetypeRegex (fileMimeType: string, mimeTypeRegex: string) {
+  return new RegExp(`^${mimeTypeRegex}$`, 'i').test(fileMimeType)
+}
+
+// ---------------------------------------------------------------------------
+
+function toCompleteUUID (value: string) {
+  if (isShortUUID(value)) {
+    try {
+      return shortToUUID(value)
+    } catch {
+      return ''
+    }
+  }
+
+  return value
+}
+
+function toCompleteUUIDs (values: string[]) {
+  return values.map(v => toCompleteUUID(v))
+}
+
+function toIntOrNull (value: string) {
+  const v = toValueOrNull(value)
+
+  if (v === null || v === undefined) return v
+  if (typeof v === 'number') return v
+
+  return validator.default.toInt('' + v)
+}
+
+function toBooleanOrNull (value: any) {
+  const v = toValueOrNull(value)
+
+  if (v === null || v === undefined) return v
+  if (typeof v === 'boolean') return v
+
+  return validator.default.toBoolean('' + v)
+}
+
+function toValueOrNull (value: string) {
+  if (value === 'null') return null
+
+  return value
+}
+
+function toIntArray (value: any) {
+  if (!value) return []
+  if (isArray(value) === false) return [ validator.default.toInt(value) ]
+
+  return value.map(v => validator.default.toInt(v))
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  exists,
+  isArrayOf,
+  isNotEmptyIntArray,
+  isArray,
+  isIntOrNull,
+  isIdValid,
+  isSafePath,
+  isNotEmptyStringArray,
+  isUUIDValid,
+  toCompleteUUIDs,
+  toCompleteUUID,
+  isIdOrUUIDValid,
+  isDateValid,
+  toValueOrNull,
+  toBooleanOrNull,
+  isBooleanValid,
+  toIntOrNull,
+  areUUIDsValid,
+  toIntArray,
+  isFileValid,
+  isSafePeerTubeFilenameWithoutExtension,
+  isSafeFilename,
+  checkMimetypeRegex
+}

server/core/helpers/custom-validators/plugins.ts

@@ -0,0 +1,177 @@
+import validator from 'validator'
+import { PluginPackageJSON, PluginType, PluginType_Type } from '@peertube/peertube-models'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { isUrlValid } from './activitypub/misc.js'
+import { exists, isArray, isSafePath } from './misc.js'
+
+const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS
+
+function isPluginTypeValid (value: any) {
+  return exists(value) &&
+    (value === PluginType.PLUGIN || value === PluginType.THEME)
+}
+
+function isPluginNameValid (value: string) {
+  return exists(value) &&
+    validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
+    validator.default.matches(value, /^[a-z-0-9]+$/)
+}
+
+function isNpmPluginNameValid (value: string) {
+  return exists(value) &&
+    validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
+    validator.default.matches(value, /^[a-z\-._0-9]+$/) &&
+    (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-'))
+}
+
+function isPluginDescriptionValid (value: string) {
+  return exists(value) && validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+function isPluginStableVersionValid (value: string) {
+  if (!exists(value)) return false
+
+  const parts = (value + '').split('.')
+
+  return parts.length === 3 && parts.every(p => validator.default.isInt(p))
+}
+
+function isPluginStableOrUnstableVersionValid (value: string) {
+  if (!exists(value)) return false
+
+  // suffix is beta.x or alpha.x
+  const [ stable, suffix ] = value.split('-')
+  if (!isPluginStableVersionValid(stable)) return false
+
+  const suffixRegex = /^(rc|alpha|beta)\.\d+$/
+  if (suffix && !suffixRegex.test(suffix)) return false
+
+  return true
+}
+
+function isPluginEngineValid (engine: any) {
+  return exists(engine) && exists(engine.peertube)
+}
+
+function isPluginHomepage (value: string) {
+  return exists(value) && (!value || isUrlValid(value))
+}
+
+function isPluginBugs (value: string) {
+  return exists(value) && (!value || isUrlValid(value))
+}
+
+function areStaticDirectoriesValid (staticDirs: any) {
+  if (!exists(staticDirs) || typeof staticDirs !== 'object') return false
+
+  for (const key of Object.keys(staticDirs)) {
+    if (!isSafePath(staticDirs[key])) return false
+  }
+
+  return true
+}
+
+function areClientScriptsValid (clientScripts: any[]) {
+  return isArray(clientScripts) &&
+    clientScripts.every(c => {
+      return isSafePath(c.script) && isArray(c.scopes)
+    })
+}
+
+function areTranslationPathsValid (translations: any) {
+  if (!exists(translations) || typeof translations !== 'object') return false
+
+  for (const key of Object.keys(translations)) {
+    if (!isSafePath(translations[key])) return false
+  }
+
+  return true
+}
+
+function areCSSPathsValid (css: any[]) {
+  return isArray(css) && css.every(c => isSafePath(c))
+}
+
+function isThemeNameValid (name: string) {
+  return isPluginNameValid(name)
+}
+
+function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType_Type) {
+  let result = true
+  const badFields: string[] = []
+
+  if (!isNpmPluginNameValid(packageJSON.name)) {
+    result = false
+    badFields.push('name')
+  }
+
+  if (!isPluginDescriptionValid(packageJSON.description)) {
+    result = false
+    badFields.push('description')
+  }
+
+  if (!isPluginEngineValid(packageJSON.engine)) {
+    result = false
+    badFields.push('engine')
+  }
+
+  if (!isPluginHomepage(packageJSON.homepage)) {
+    result = false
+    badFields.push('homepage')
+  }
+
+  if (!exists(packageJSON.author)) {
+    result = false
+    badFields.push('author')
+  }
+
+  if (!isPluginBugs(packageJSON.bugs)) {
+    result = false
+    badFields.push('bugs')
+  }
+
+  if (pluginType === PluginType.PLUGIN && !isSafePath(packageJSON.library)) {
+    result = false
+    badFields.push('library')
+  }
+
+  if (!areStaticDirectoriesValid(packageJSON.staticDirs)) {
+    result = false
+    badFields.push('staticDirs')
+  }
+
+  if (!areCSSPathsValid(packageJSON.css)) {
+    result = false
+    badFields.push('css')
+  }
+
+  if (!areClientScriptsValid(packageJSON.clientScripts)) {
+    result = false
+    badFields.push('clientScripts')
+  }
+
+  if (!areTranslationPathsValid(packageJSON.translations)) {
+    result = false
+    badFields.push('translations')
+  }
+
+  return { result, badFields }
+}
+
+function isLibraryCodeValid (library: any) {
+  return typeof library.register === 'function' &&
+    typeof library.unregister === 'function'
+}
+
+export {
+  isPluginTypeValid,
+  isPackageJSONValid,
+  isThemeNameValid,
+  isPluginHomepage,
+  isPluginStableVersionValid,
+  isPluginStableOrUnstableVersionValid,
+  isPluginNameValid,
+  isPluginDescriptionValid,
+  isLibraryCodeValid,
+  isNpmPluginNameValid
+}

server/core/helpers/custom-validators/runners/jobs.ts

@@ -0,0 +1,206 @@
+import {
+  LiveRTMPHLSTranscodingSuccess,
+  RunnerJobSuccessPayload,
+  RunnerJobType,
+  RunnerJobUpdatePayload,
+  TranscriptionSuccess,
+  VODAudioMergeTranscodingSuccess,
+  VODHLSTranscodingSuccess,
+  VODWebVideoTranscodingSuccess,
+  VideoStudioTranscodingSuccess
+} from '@peertube/peertube-models'
+import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js'
+import { UploadFilesForCheck } from 'express'
+import validator from 'validator'
+import { exists, isArray, isFileValid, isSafeFilename } from '../misc.js'
+
+const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
+
+const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ])
+export function isRunnerJobTypeValid (value: RunnerJobType) {
+  return runnerJobTypes.has(value)
+}
+
+export function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) {
+  return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) ||
+    isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
+    isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
+    isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) ||
+    isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) ||
+    isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files)
+}
+
+// ---------------------------------------------------------------------------
+
+export function isRunnerJobProgressValid (value: string) {
+  return validator.default.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS)
+}
+
+export function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) {
+  return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVODHLSUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||
+    isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) ||
+    isRunnerJobTranscriptionUpdatePayloadValid(value, type, files)
+}
+
+// ---------------------------------------------------------------------------
+
+export function isRunnerJobTokenValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+export function isRunnerJobAbortReasonValid (value: string) {
+  return validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON)
+}
+
+export function isRunnerJobErrorMessageValid (value: string) {
+  return validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE)
+}
+
+export function isRunnerJobStateValid (value: any) {
+  return exists(value) && RUNNER_JOB_STATES[value] !== undefined
+}
+
+export function isRunnerJobArrayOfStateValid (value: any) {
+  return isArray(value) && value.every(v => isRunnerJobStateValid(v))
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function isRunnerJobVODWebVideoResultPayloadValid (
+  _value: VODWebVideoTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-web-video-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobVODHLSResultPayloadValid (
+  _value: VODHLSTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-hls-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) &&
+    isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobVODAudioMergeResultPayloadValid (
+  _value: VODAudioMergeTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-audio-merge-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobLiveRTMPHLSResultPayloadValid (
+  value: LiveRTMPHLSTranscodingSuccess,
+  type: RunnerJobType
+) {
+  return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobVideoStudioResultPayloadValid (
+  _value: VideoStudioTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'video-studio-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobTranscriptionResultPayloadValid (
+  value: TranscriptionSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'video-transcription' &&
+    isFileValid({ files, field: 'payload[vttFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+// ---------------------------------------------------------------------------
+
+function isRunnerJobVODWebVideoUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-web-video-transcoding' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobVODHLSUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-hls-transcoding' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobVODAudioMergeUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-audio-merge-transcoding' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobTranscriptionUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'video-transcription' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobLiveRTMPHLSUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files
+
+  result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true })
+
+  result &&= isFileValid({
+    files,
+    field: 'payload[resolutionPlaylistFile]',
+    mimeTypeRegex: null,
+    maxSize: null,
+    optional: !value.resolutionPlaylistFilename
+  })
+
+  if (files['payload[resolutionPlaylistFile]']) {
+    result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8')
+  }
+
+  return result &&
+    isSafeFilename(value.videoChunkFilename, 'ts') &&
+    (
+      (
+        value.type === 'remove-chunk'
+      ) ||
+      (
+        value.type === 'add-chunk' &&
+        isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null })
+      )
+    )
+}
+
+function isRunnerJobVideoStudioUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'video-studio-transcoding' &&
+    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}

server/core/helpers/custom-validators/runners/runners.ts

@@ -0,0 +1,30 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import { exists } from '../misc.js'
+
+const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS
+
+function isRunnerRegistrationTokenValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+function isRunnerTokenValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+function isRunnerNameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME)
+}
+
+function isRunnerDescriptionValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isRunnerRegistrationTokenValid,
+  isRunnerTokenValid,
+  isRunnerNameValid,
+  isRunnerDescriptionValid
+}

server/core/helpers/custom-validators/search.ts

@@ -0,0 +1,37 @@
+import validator from 'validator'
+import { SearchTargetType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { exists, isArray } from './misc.js'
+
+function isNumberArray (value: any) {
+  return isArray(value) && value.every(v => validator.default.isInt('' + v))
+}
+
+function isStringArray (value: any) {
+  return isArray(value) && value.every(v => typeof v === 'string')
+}
+
+function isBooleanBothQueryValid (value: any) {
+  return value === 'true' || value === 'false' || value === 'both'
+}
+
+function isSearchTargetValid (value: SearchTargetType) {
+  if (!exists(value)) return true
+
+  const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX
+
+  if (value === 'local') return true
+
+  if (value === 'search-index' && searchIndexConfig.ENABLED) return true
+
+  return false
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isNumberArray,
+  isStringArray,
+  isBooleanBothQueryValid,
+  isSearchTargetValid
+}

server/core/helpers/custom-validators/servers.ts

@@ -0,0 +1,42 @@
+import validator from 'validator'
+import { CONFIG } from '@server/initializers/config.js'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists, isArray } from './misc.js'
+
+function isHostValid (host: string) {
+  const isURLOptions = {
+    require_host: true,
+    require_tld: true
+  }
+
+  // We validate 'localhost', so we don't have the top level domain
+  if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') {
+    isURLOptions.require_tld = false
+  }
+
+  return exists(host) && validator.default.isURL(host, isURLOptions) && host.split('://').length === 1
+}
+
+function isEachUniqueHostValid (hosts: string[]) {
+  return isArray(hosts) &&
+    hosts.every(host => {
+      return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host)
+    })
+}
+
+function isValidContactBody (value: any) {
+  return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.BODY)
+}
+
+function isValidContactFromName (value: any) {
+  return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.FROM_NAME)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidContactBody,
+  isValidContactFromName,
+  isEachUniqueHostValid,
+  isHostValid
+}

server/core/helpers/custom-validators/user-notifications.ts

@@ -0,0 +1,23 @@
+import validator from 'validator'
+import { UserNotificationSettingValue } from '@peertube/peertube-models'
+import { exists } from './misc.js'
+
+function isUserNotificationTypeValid (value: any) {
+  return exists(value) && validator.default.isInt('' + value)
+}
+
+function isUserNotificationSettingValid (value: any) {
+  return exists(value) &&
+    validator.default.isInt('' + value) &&
+    (
+      value === UserNotificationSettingValue.NONE ||
+      value === UserNotificationSettingValue.WEB ||
+      value === UserNotificationSettingValue.EMAIL ||
+      value === (UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL)
+    )
+}
+
+export {
+  isUserNotificationSettingValid,
+  isUserNotificationTypeValid
+}

server/core/helpers/custom-validators/user-registration.ts

@@ -0,0 +1,25 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, USER_REGISTRATION_STATES } from '../../initializers/constants.js'
+import { exists } from './misc.js'
+
+const USER_REGISTRATIONS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USER_REGISTRATIONS
+
+function isRegistrationStateValid (value: string) {
+  return exists(value) && USER_REGISTRATION_STATES[value] !== undefined
+}
+
+function isRegistrationModerationResponseValid (value: string) {
+  return exists(value) && validator.default.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.MODERATOR_MESSAGE)
+}
+
+function isRegistrationReasonValid (value: string) {
+  return exists(value) && validator.default.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.REASON_MESSAGE)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isRegistrationStateValid,
+  isRegistrationModerationResponseValid,
+  isRegistrationReasonValid
+}

server/core/helpers/custom-validators/users.ts

@@ -0,0 +1,125 @@
+import validator from 'validator'
+import { UserRole } from '@peertube/peertube-models'
+import { isEmailEnabled } from '../../initializers/config.js'
+import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants.js'
+import { exists, isArray, isBooleanValid } from './misc.js'
+
+const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS
+
+function isUserPasswordValid (value: string) {
+  return validator.default.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD)
+}
+
+function isUserPasswordValidOrEmpty (value: string) {
+  // Empty password is only possible if emailing is enabled.
+  if (value === '') return isEmailEnabled()
+
+  return isUserPasswordValid(value)
+}
+
+function isUserVideoQuotaValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA)
+}
+
+function isUserVideoQuotaDailyValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY)
+}
+
+function isUserUsernameValid (value: string) {
+  return exists(value) &&
+    validator.default.matches(value, new RegExp(`^[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?$`)) &&
+    validator.default.isLength(value, USERS_CONSTRAINTS_FIELDS.USERNAME)
+}
+
+function isUserDisplayNameValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.NAME))
+}
+
+function isUserDescriptionValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.DESCRIPTION))
+}
+
+function isUserEmailVerifiedValid (value: any) {
+  return isBooleanValid(value)
+}
+
+const nsfwPolicies = new Set(Object.values(NSFW_POLICY_TYPES))
+function isUserNSFWPolicyValid (value: any) {
+  return exists(value) && nsfwPolicies.has(value)
+}
+
+function isUserP2PEnabledValid (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserVideosHistoryEnabledValid (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserAutoPlayVideoValid (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserVideoLanguages (value: any) {
+  return value === null || (isArray(value) && value.length < CONSTRAINTS_FIELDS.USERS.VIDEO_LANGUAGES.max)
+}
+
+function isUserAdminFlagsValid (value: any) {
+  return exists(value) && validator.default.isInt('' + value)
+}
+
+function isUserBlockedValid (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserAutoPlayNextVideoValid (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserAutoPlayNextVideoPlaylistValid (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserEmailPublicValid (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserNoModal (value: any) {
+  return isBooleanValid(value)
+}
+
+function isUserBlockedReasonValid (value: any) {
+  return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.BLOCKED_REASON))
+}
+
+function isUserRoleValid (value: any) {
+  return exists(value) &&
+    validator.default.isInt('' + value) &&
+    [ UserRole.ADMINISTRATOR, UserRole.MODERATOR, UserRole.USER ].includes(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isUserVideosHistoryEnabledValid,
+  isUserBlockedValid,
+  isUserPasswordValid,
+  isUserPasswordValidOrEmpty,
+  isUserVideoLanguages,
+  isUserBlockedReasonValid,
+  isUserRoleValid,
+  isUserVideoQuotaValid,
+  isUserVideoQuotaDailyValid,
+  isUserUsernameValid,
+  isUserAdminFlagsValid,
+  isUserEmailVerifiedValid,
+  isUserNSFWPolicyValid,
+  isUserP2PEnabledValid,
+  isUserAutoPlayVideoValid,
+  isUserAutoPlayNextVideoValid,
+  isUserAutoPlayNextVideoPlaylistValid,
+  isUserDisplayNameValid,
+  isUserDescriptionValid,
+  isUserEmailPublicValid,
+  isUserNoModal
+}

server/core/helpers/custom-validators/video-blacklist.ts

@@ -0,0 +1,22 @@
+import validator from 'validator'
+import { VideoBlacklistType } from '@peertube/peertube-models'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists } from './misc.js'
+
+const VIDEO_BLACKLIST_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_BLACKLIST
+
+function isVideoBlacklistReasonValid (value: string) {
+  return value === null || validator.default.isLength(value, VIDEO_BLACKLIST_CONSTRAINTS_FIELDS.REASON)
+}
+
+function isVideoBlacklistTypeValid (value: any) {
+  return exists(value) &&
+    (value === VideoBlacklistType.AUTO_BEFORE_PUBLISHED || value === VideoBlacklistType.MANUAL)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoBlacklistReasonValid,
+  isVideoBlacklistTypeValid
+}

server/core/helpers/custom-validators/video-captions.ts

@@ -0,0 +1,43 @@
+import { UploadFilesForCheck } from 'express'
+import { readFile } from 'fs/promises'
+import { getFileSize } from '@peertube/peertube-node-utils'
+import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants.js'
+import { logger } from '../logger.js'
+import { exists, isFileValid } from './misc.js'
+
+function isVideoCaptionLanguageValid (value: any) {
+  return exists(value) && VIDEO_LANGUAGES[value] !== undefined
+}
+
+// MacOS sends application/octet-stream
+const videoCaptionTypesRegex = [ ...Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT), 'application/octet-stream' ]
+  .map(m => `(${m})`)
+  .join('|')
+
+function isVideoCaptionFile (files: UploadFilesForCheck, field: string) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: videoCaptionTypesRegex,
+    field,
+    maxSize: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max
+  })
+}
+
+async function isVTTFileValid (filePath: string) {
+  const size = await getFileSize(filePath)
+  const content = await readFile(filePath, 'utf8')
+
+  logger.debug('Checking VTT file %s', filePath, { size, content })
+
+  if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false
+
+  return content?.startsWith('WEBVTT')
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoCaptionFile,
+  isVTTFileValid,
+  isVideoCaptionLanguageValid
+}

server/core/helpers/custom-validators/video-channel-syncs.ts

@@ -0,0 +1,6 @@
+import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants.js'
+import { exists } from './misc.js'
+
+export function isVideoChannelSyncStateValid (value: any) {
+  return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
+}

server/core/helpers/custom-validators/video-channels.ts

@@ -0,0 +1,32 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists } from './misc.js'
+import { isUserUsernameValid } from './users.js'
+
+const VIDEO_CHANNELS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_CHANNELS
+
+function isVideoChannelUsernameValid (value: string) {
+  // Use the same constraints than user username
+  return isUserUsernameValid(value)
+}
+
+function isVideoChannelDescriptionValid (value: string) {
+  return value === null || validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+function isVideoChannelDisplayNameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.NAME)
+}
+
+function isVideoChannelSupportValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.SUPPORT))
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoChannelUsernameValid,
+  isVideoChannelDescriptionValid,
+  isVideoChannelDisplayNameValid,
+  isVideoChannelSupportValid
+}

server/core/helpers/custom-validators/video-chapters.ts

@@ -0,0 +1,26 @@
+import { isArray } from './misc.js'
+import { VideoChapter, VideoChapterUpdate } from '@peertube/peertube-models'
+import { Unpacked } from '@peertube/peertube-typescript-utils'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import validator from 'validator'
+
+export function areVideoChaptersValid (value: VideoChapter[]) {
+  if (!isArray(value)) return false
+  if (!value.every(v => isVideoChapterValid(v))) return false
+
+  const timecodes = value.map(c => c.timecode)
+
+  return new Set(timecodes).size === timecodes.length
+}
+
+export function isVideoChapterValid (value: Unpacked<VideoChapterUpdate['chapters']>) {
+  return isVideoChapterTimecodeValid(value.timecode) && isVideoChapterTitleValid(value.title)
+}
+
+export function isVideoChapterTitleValid (value: any) {
+  return validator.default.isLength(value + '', CONSTRAINTS_FIELDS.VIDEO_CHAPTERS.TITLE)
+}
+
+export function isVideoChapterTimecodeValid (value: any) {
+  return validator.default.isInt(value + '', { min: 0 })
+}

server/core/helpers/custom-validators/video-comments.ts

@@ -0,0 +1,14 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+
+const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS
+
+function isValidVideoCommentText (value: string) {
+  return value === null || validator.default.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidVideoCommentText
+}

server/core/helpers/custom-validators/video-imports.ts

@@ -0,0 +1,46 @@
+import 'multer'
+import { UploadFilesForCheck } from 'express'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants.js'
+import { exists, isFileValid } from './misc.js'
+
+function isVideoImportTargetUrlValid (url: string) {
+  const isURLOptions = {
+    require_host: true,
+    require_tld: true,
+    require_protocol: true,
+    require_valid_protocol: true,
+    protocols: [ 'http', 'https' ]
+  }
+
+  return exists(url) &&
+    validator.default.isURL('' + url, isURLOptions) &&
+    validator.default.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
+}
+
+function isVideoImportStateValid (value: any) {
+  return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined
+}
+
+// MacOS sends application/octet-stream
+const videoTorrentImportRegex = [ ...Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT), 'application/octet-stream' ]
+  .map(m => `(${m})`)
+  .join('|')
+
+function isVideoImportTorrentFile (files: UploadFilesForCheck) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: videoTorrentImportRegex,
+    field: 'torrentfile',
+    maxSize: CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max,
+    optional: true
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoImportStateValid,
+  isVideoImportTargetUrlValid,
+  isVideoImportTorrentFile
+}

server/core/helpers/custom-validators/video-lives.ts

@@ -0,0 +1,11 @@
+import { LiveVideoLatencyMode } from '@peertube/peertube-models'
+
+function isLiveLatencyModeValid (value: any) {
+  return [ LiveVideoLatencyMode.DEFAULT, LiveVideoLatencyMode.SMALL_LATENCY, LiveVideoLatencyMode.HIGH_LATENCY ].includes(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isLiveLatencyModeValid
+}

server/core/helpers/custom-validators/video-ownership.ts

@@ -0,0 +1,20 @@
+import { Response } from 'express'
+import { HttpStatusCode } from '@peertube/peertube-models'
+import { MUserId } from '@server/types/models/index.js'
+import { MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership.js'
+
+function checkUserCanTerminateOwnershipChange (user: MUserId, videoChangeOwnership: MVideoChangeOwnershipFull, res: Response) {
+  if (videoChangeOwnership.NextOwner.userId === user.id) {
+    return true
+  }
+
+  res.fail({
+    status: HttpStatusCode.FORBIDDEN_403,
+    message: 'Cannot terminate an ownership change of another user'
+  })
+  return false
+}
+
+export {
+  checkUserCanTerminateOwnershipChange
+}

server/core/helpers/custom-validators/video-playlists.ts

@@ -0,0 +1,35 @@
+import { exists } from './misc.js'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants.js'
+
+const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS
+
+function isVideoPlaylistNameValid (value: any) {
+  return exists(value) && validator.default.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.NAME)
+}
+
+function isVideoPlaylistDescriptionValid (value: any) {
+  return value === null || (exists(value) && validator.default.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.DESCRIPTION))
+}
+
+function isVideoPlaylistPrivacyValid (value: number) {
+  return validator.default.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined
+}
+
+function isVideoPlaylistTimestampValid (value: any) {
+  return value === null || (exists(value) && validator.default.isInt('' + value, { min: 0 }))
+}
+
+function isVideoPlaylistTypeValid (value: any) {
+  return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoPlaylistNameValid,
+  isVideoPlaylistDescriptionValid,
+  isVideoPlaylistPrivacyValid,
+  isVideoPlaylistTimestampValid,
+  isVideoPlaylistTypeValid
+}

server/core/helpers/custom-validators/video-rates.ts

@@ -0,0 +1,5 @@
+function isRatingValid (value: any) {
+  return value === 'like' || value === 'dislike'
+}
+
+export { isRatingValid }

server/core/helpers/custom-validators/video-redundancies.ts

@@ -0,0 +1,12 @@
+import { exists } from './misc.js'
+
+function isVideoRedundancyTarget (value: any) {
+  return exists(value) &&
+    (value === 'my-videos' || value === 'remote-videos')
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoRedundancyTarget
+}

server/core/helpers/custom-validators/video-stats.ts

@@ -0,0 +1,16 @@
+import { VideoStatsTimeserieMetric } from '@peertube/peertube-models'
+
+const validMetrics = new Set<VideoStatsTimeserieMetric>([
+  'viewers',
+  'aggregateWatchTime'
+])
+
+function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) {
+  return validMetrics.has(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidStatTimeserieMetric
+}

server/core/helpers/custom-validators/video-studio.ts

@@ -0,0 +1,53 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import { buildTaskFileFieldname } from '@server/lib/video-studio.js'
+import { VideoStudioTask } from '@peertube/peertube-models'
+import { isArray } from './misc.js'
+import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos.js'
+import { forceNumber } from '@peertube/peertube-core-utils'
+
+function isValidStudioTasksArray (tasks: any) {
+  if (!isArray(tasks)) return false
+
+  return tasks.length >= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.min &&
+    tasks.length <= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.max
+}
+
+function isStudioCutTaskValid (task: VideoStudioTask) {
+  if (task.name !== 'cut') return false
+  if (!task.options) return false
+
+  const { start, end } = task.options
+  if (!start && !end) return false
+
+  if (start && !validator.default.isInt(start + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false
+  if (end && !validator.default.isInt(end + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false
+
+  if (!start || !end) return true
+
+  return forceNumber(start) < forceNumber(end)
+}
+
+function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) {
+  const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file'))
+
+  return (task.name === 'add-intro' || task.name === 'add-outro') &&
+    file && isVideoFileMimeTypeValid([ file ], null)
+}
+
+function isStudioTaskAddWatermarkValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) {
+  const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file'))
+
+  return task.name === 'add-watermark' &&
+    file && isVideoImageValid([ file ], null, true)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidStudioTasksArray,
+
+  isStudioCutTaskValid,
+  isStudioTaskAddIntroOutroValid,
+  isStudioTaskAddWatermarkValid
+}

server/core/helpers/custom-validators/video-transcoding.ts

@@ -0,0 +1,12 @@
+import { exists } from './misc.js'
+
+function isValidCreateTranscodingType (value: any) {
+  return exists(value) &&
+    (value === 'hls' || value === 'webtorrent' || value === 'web-video') // TODO: remove webtorrent in v7
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidCreateTranscodingType
+}

server/core/helpers/custom-validators/video-view.ts

@@ -0,0 +1,12 @@
+import { exists } from './misc.js'
+
+function isVideoTimeValid (value: number, videoDuration?: number) {
+  if (value < 0) return false
+  if (exists(videoDuration) && value > videoDuration) return false
+
+  return true
+}
+
+export {
+  isVideoTimeValid
+}

server/core/helpers/custom-validators/videos.ts

@@ -0,0 +1,195 @@
+import { HttpStatusCode, VideoIncludeType, VideoPrivacy, VideoPrivacyType, VideoRateType } from '@peertube/peertube-models'
+import { getVideoWithAttributes } from '@server/helpers/video.js'
+import { Request, Response, UploadFilesForCheck } from 'express'
+import { decode as magnetUriDecode } from 'magnet-uri'
+import validator from 'validator'
+import {
+  CONSTRAINTS_FIELDS,
+  MIMETYPES,
+  VIDEO_CATEGORIES,
+  VIDEO_COMMENTS_POLICY,
+  VIDEO_LICENCES,
+  VIDEO_LIVE,
+  VIDEO_PRIVACIES,
+  VIDEO_RATE_TYPES,
+  VIDEO_STATES
+} from '../../initializers/constants.js'
+import { exists, isArray, isDateValid, isFileValid } from './misc.js'
+
+const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
+
+export function isVideoIncludeValid (include: VideoIncludeType) {
+  return exists(include) && validator.default.isInt('' + include)
+}
+
+export function isVideoCategoryValid (value: any) {
+  return value === null || VIDEO_CATEGORIES[value] !== undefined
+}
+
+export function isVideoStateValid (value: any) {
+  return exists(value) && VIDEO_STATES[value] !== undefined
+}
+
+export function isVideoLicenceValid (value: any) {
+  return value === null || VIDEO_LICENCES[value] !== undefined
+}
+
+export function isVideoLanguageValid (value: any) {
+  return value === null ||
+    (typeof value === 'string' && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.LANGUAGE))
+}
+
+export function isVideoDurationValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION)
+}
+
+export function isVideoDescriptionValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION))
+}
+
+export function isVideoCommentsPolicyValid (value: any) {
+  return value === null || VIDEO_COMMENTS_POLICY[value] !== undefined
+}
+
+export function isVideoSupportValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.SUPPORT))
+}
+
+export function isVideoNameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME)
+}
+
+export function isVideoSourceFilenameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.VIDEO_SOURCE.FILENAME)
+}
+
+export function isVideoTagValid (tag: string) {
+  return exists(tag) && validator.default.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG)
+}
+
+export function areVideoTagsValid (tags: string[]) {
+  return tags === null || (
+    isArray(tags) &&
+    validator.default.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) &&
+    tags.every(tag => isVideoTagValid(tag))
+  )
+}
+
+export function isVideoViewsValid (value: string | number) {
+  return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS)
+}
+
+const ratingTypes = new Set(Object.values(VIDEO_RATE_TYPES))
+export function isVideoRatingTypeValid (value: string) {
+  return value === 'none' || ratingTypes.has(value as VideoRateType)
+}
+
+export function isVideoFileExtnameValid (value: string) {
+  return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
+}
+
+export function isVideoFileMimeTypeValid (files: UploadFilesForCheck, field = 'videofile') {
+  return isFileValid({
+    files,
+    mimeTypeRegex: MIMETYPES.VIDEO.MIMETYPES_REGEX,
+    field,
+    maxSize: null
+  })
+}
+
+const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME
+                                          .map(v => v.replace('.', ''))
+                                          .join('|')
+const videoImageTypesRegex = `image/(${videoImageTypes})`
+
+export function isVideoImageValid (files: UploadFilesForCheck, field: string, optional = true) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: videoImageTypesRegex,
+    field,
+    maxSize: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max,
+    optional
+  })
+}
+
+export function isVideoPrivacyValid (value: number) {
+  return VIDEO_PRIVACIES[value] !== undefined
+}
+
+export function isVideoReplayPrivacyValid (value: number) {
+  return VIDEO_PRIVACIES[value] !== undefined && value !== VideoPrivacy.PASSWORD_PROTECTED
+}
+
+export function isScheduleVideoUpdatePrivacyValid (value: number) {
+  return value === VideoPrivacy.UNLISTED || value === VideoPrivacy.PUBLIC || value === VideoPrivacy.INTERNAL
+}
+
+export function isVideoOriginallyPublishedAtValid (value: string | null) {
+  return value === null || isDateValid(value)
+}
+
+export function isVideoFileInfoHashValid (value: string | null | undefined) {
+  return exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH)
+}
+
+export function isVideoFileResolutionValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '')
+}
+
+export function isVideoFPSResolutionValid (value: string) {
+  return value === null || validator.default.isInt(value + '')
+}
+
+export function isVideoFileSizeValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE)
+}
+
+export function isVideoMagnetUriValid (value: string) {
+  if (!exists(value)) return false
+
+  const parsed = magnetUriDecode(value)
+  return parsed && isVideoFileInfoHashValid(parsed.infoHash)
+}
+
+export function isPasswordValid (password: string) {
+  return password.length >= CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.min &&
+    password.length < CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.max
+}
+
+export function isValidPasswordProtectedPrivacy (req: Request, res: Response) {
+  const fail = (message: string) => {
+    res.fail({
+      status: HttpStatusCode.BAD_REQUEST_400,
+      message
+    })
+    return false
+  }
+
+  let privacy: VideoPrivacyType
+  const video = getVideoWithAttributes(res)
+
+  if (exists(req.body?.privacy)) privacy = req.body.privacy
+  else if (exists(video?.privacy)) privacy = video.privacy
+
+  if (privacy !== VideoPrivacy.PASSWORD_PROTECTED) return true
+
+  if (!exists(req.body.videoPasswords) && !exists(req.body.passwords)) return fail('Video passwords are missing.')
+
+  const passwords = req.body.videoPasswords || req.body.passwords
+
+  if (passwords.length === 0) return fail('At least one video password is required.')
+
+  if (new Set(passwords).size !== passwords.length) return fail('Duplicate video passwords are not allowed.')
+
+  for (const password of passwords) {
+    if (typeof password !== 'string') {
+      return fail('Video password should be a string.')
+    }
+
+    if (!isPasswordValid(password)) {
+      return fail('Invalid video password. Password length should be at least 2 characters and no more than 100 characters.')
+    }
+  }
+
+  return true
+}

server/core/helpers/custom-validators/watched-words.ts

@@ -0,0 +1,17 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists, isArray } from './misc.js'
+
+export function isWatchedWordListNameValid (listName: string) {
+  return exists(listName) && validator.default.isLength(listName, CONSTRAINTS_FIELDS.WATCHED_WORDS.LIST_NAME)
+}
+
+export function isWatchedWordValid (word: string) {
+  return exists(word) && validator.default.isLength(word, CONSTRAINTS_FIELDS.WATCHED_WORDS.WORD)
+}
+
+export function areWatchedWordsValid (words: string[]) {
+  return isArray(words) &&
+    validator.default.isInt(words.length.toString(), CONSTRAINTS_FIELDS.WATCHED_WORDS.WORDS) &&
+    words.every(word => isWatchedWordValid(word))
+}

server/core/helpers/custom-validators/webfinger.ts

@@ -0,0 +1,21 @@
+import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants.js'
+import { sanitizeHost } from '../core-utils.js'
+import { exists } from './misc.js'
+
+function isWebfingerLocalResourceValid (value: string) {
+  if (!exists(value)) return false
+  if (value.startsWith('acct:') === false) return false
+
+  const actorWithHost = value.substr(5)
+  const actorParts = actorWithHost.split('@')
+  if (actorParts.length !== 2) return false
+
+  const host = actorParts[1]
+  return sanitizeHost(host, REMOTE_SCHEME.HTTP) === WEBSERVER.HOST
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isWebfingerLocalResourceValid
+}

server/core/helpers/database-utils.ts

@@ -0,0 +1,121 @@
+import retry from 'async/retry.js'
+import Bluebird from 'bluebird'
+import { Transaction } from 'sequelize'
+import { Model } from 'sequelize-typescript'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { logger } from './logger.js'
+
+function retryTransactionWrapper <T, A, B, C, D> (
+  functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T>,
+  arg1: A,
+  arg2: B,
+  arg3: C,
+  arg4: D,
+): Promise<T>
+
+function retryTransactionWrapper <T, A, B, C> (
+  functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T>,
+  arg1: A,
+  arg2: B,
+  arg3: C
+): Promise<T>
+
+function retryTransactionWrapper <T, A, B> (
+  functionToRetry: (arg1: A, arg2: B) => Promise<T>,
+  arg1: A,
+  arg2: B
+): Promise<T>
+
+function retryTransactionWrapper <T, A> (
+  functionToRetry: (arg1: A) => Promise<T>,
+  arg1: A
+): Promise<T>
+
+function retryTransactionWrapper <T> (
+  functionToRetry: () => Promise<T> | Bluebird<T>
+): Promise<T>
+
+function retryTransactionWrapper <T> (
+  functionToRetry: (...args: any[]) => Promise<T>,
+  ...args: any[]
+): Promise<T> {
+  return transactionRetryer<T>(callback => {
+    functionToRetry.apply(null, args)
+        .then((result: T) => callback(null, result))
+        .catch(err => callback(err))
+  })
+  .catch(err => {
+    logger.warn(`Cannot execute ${functionToRetry.name} with many retries.`, { err })
+    throw err
+  })
+}
+
+function transactionRetryer <T> (func: (err: any, data: T) => any) {
+  return new Promise<T>((res, rej) => {
+    retry(
+      {
+        times: 5,
+
+        errorFilter: err => {
+          const willRetry = (err.name === 'SequelizeDatabaseError')
+          logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })
+          return willRetry
+        }
+      },
+      func,
+      (err, data) => err ? rej(err) : res(data)
+    )
+  })
+}
+
+function saveInTransactionWithRetries <T extends Pick<Model, 'save'>> (model: T) {
+  return retryTransactionWrapper(() => {
+    return sequelizeTypescript.transaction(async transaction => {
+      await model.save({ transaction })
+    })
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function resetSequelizeInstance <T> (instance: Model<T>) {
+  return instance.reload()
+}
+
+function filterNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean }> (
+  fromDatabase: T[],
+  newModels: T[]
+) {
+  return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f)))
+}
+
+function deleteAllModels <T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) {
+  return Promise.all(models.map(f => f.destroy({ transaction })))
+}
+
+// ---------------------------------------------------------------------------
+
+function runInReadCommittedTransaction <T> (fn: (t: Transaction) => Promise<T>) {
+  const options = { isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED }
+
+  return sequelizeTypescript.transaction(options, t => fn(t))
+}
+
+function afterCommitIfTransaction (t: Transaction, fn: Function) {
+  if (t) return t.afterCommit(() => fn())
+
+  return fn()
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  resetSequelizeInstance,
+  retryTransactionWrapper,
+  transactionRetryer,
+  saveInTransactionWithRetries,
+  afterCommitIfTransaction,
+  filterNonExistingModels,
+  deleteAllModels,
+  runInReadCommittedTransaction
+}

server/core/helpers/debounce.ts

@@ -0,0 +1,15 @@
+export function Debounce (config: { timeoutMS: number }) {
+  let timeoutRef: NodeJS.Timeout
+
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const original = descriptor.value
+
+    descriptor.value = function (...args: any[]) {
+      clearTimeout(timeoutRef)
+
+      timeoutRef = setTimeout(() => {
+        original.apply(this, args)
+      }, config.timeoutMS)
+    }
+  }
+}

server/core/helpers/decache.ts

@@ -0,0 +1,79 @@
+// Thanks: https://github.com/dwyl/decache
+// We reuse this file to also uncache plugin base path
+
+import { Module } from 'module'
+import { extname } from 'path'
+
+function decachePlugin (require: NodeRequire, libraryPath: string) {
+  const moduleName = find(require, libraryPath)
+
+  if (!moduleName) return
+
+  searchCache(require, moduleName, function (mod) {
+    delete require.cache[mod.id]
+
+    removeCachedPath(mod.path)
+  })
+}
+
+function decacheModule (require: NodeRequire, name: string) {
+  const moduleName = find(require, name)
+
+  if (!moduleName) return
+
+  searchCache(require, moduleName, function (mod) {
+    delete require.cache[mod.id]
+
+    removeCachedPath(mod.path)
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  decacheModule,
+  decachePlugin
+}
+
+// ---------------------------------------------------------------------------
+
+function find (require: NodeRequire, moduleName: string) {
+  try {
+    return require.resolve(moduleName)
+  } catch {
+    return ''
+  }
+}
+
+function searchCache (require: NodeRequire, moduleName: string, callback: (current: NodeModule) => void) {
+  const resolvedModule = require.resolve(moduleName)
+  let mod: NodeModule
+  const visited = {}
+
+  if (resolvedModule && ((mod = require.cache[resolvedModule]) !== undefined)) {
+    // Recursively go over the results
+    (function run (current) {
+      visited[current.id] = true
+
+      current.children.forEach(function (child) {
+        if (extname(child.filename) !== '.node' && !visited[child.id]) {
+          run(child)
+        }
+      })
+
+      // Call the specified callback providing the
+      // found module
+      callback(current)
+    })(mod)
+  }
+};
+
+function removeCachedPath (pluginPath: string) {
+  const pathCache = (Module as any)._pathCache as { [ id: string ]: string[] }
+
+  Object.keys(pathCache).forEach(function (cacheKey) {
+    if (cacheKey.includes(pluginPath)) {
+      delete pathCache[cacheKey]
+    }
+  })
+}

server/core/helpers/dns.ts

@@ -0,0 +1,29 @@
+import { lookup } from 'dns'
+import ipaddr from 'ipaddr.js'
+
+function dnsLookupAll (hostname: string) {
+  return new Promise<string[]>((res, rej) => {
+    lookup(hostname, { family: 0, all: true }, (err, adresses) => {
+      if (err) return rej(err)
+
+      return res(adresses.map(a => a.address))
+    })
+  })
+}
+
+async function isResolvingToUnicastOnly (hostname: string) {
+  const addresses = await dnsLookupAll(hostname)
+
+  for (const address of addresses) {
+    const parsed = ipaddr.parse(address)
+
+    if (parsed.range() !== 'unicast') return false
+  }
+
+  return true
+}
+
+export {
+  dnsLookupAll,
+  isResolvingToUnicastOnly
+}

server/core/helpers/express-utils.ts

@@ -0,0 +1,156 @@
+import express, { RequestHandler } from 'express'
+import multer, { diskStorage } from 'multer'
+import { getLowercaseExtension } from '@peertube/peertube-node-utils'
+import { CONFIG } from '../initializers/config.js'
+import { REMOTE_SCHEME } from '../initializers/constants.js'
+import { isArray } from './custom-validators/misc.js'
+import { logger } from './logger.js'
+import { deleteFileAndCatch, generateRandomString } from './utils.js'
+import { getExtFromMimetype } from './video.js'
+
+function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
+  if (paramNSFW === 'true') return true
+  if (paramNSFW === 'false') return false
+  if (paramNSFW === 'both') return undefined
+
+  if (res?.locals.oauth) {
+    const user = res.locals.oauth.token.User
+
+    // User does not want NSFW videos
+    if (user.nsfwPolicy === 'do_not_list') return false
+
+    // Both
+    return undefined
+  }
+
+  if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false
+
+  // Display all
+  return null
+}
+
+function cleanUpReqFiles (req: express.Request) {
+  const filesObject = req.files
+  if (!filesObject) return
+
+  if (isArray(filesObject)) {
+    filesObject.forEach(f => deleteFileAndCatch(f.path))
+    return
+  }
+
+  for (const key of Object.keys(filesObject)) {
+    const files = filesObject[key]
+
+    files.forEach(f => deleteFileAndCatch(f.path))
+  }
+}
+
+function getHostWithPort (host: string) {
+  const splitted = host.split(':')
+
+  // The port was not specified
+  if (splitted.length === 1) {
+    if (REMOTE_SCHEME.HTTP === 'https') return host + ':443'
+
+    return host + ':80'
+  }
+
+  return host
+}
+
+function createReqFiles (
+  fieldNames: string[],
+  mimeTypes: { [id: string]: string | string[] },
+  destination = CONFIG.STORAGE.TMP_DIR
+): RequestHandler {
+  const storage = diskStorage({
+    destination: (req, file, cb) => {
+      cb(null, destination)
+    },
+
+    filename: (req, file, cb) => {
+      return generateReqFilename(file, mimeTypes, cb)
+    }
+  })
+
+  const fields: { name: string, maxCount: number }[] = []
+  for (const fieldName of fieldNames) {
+    fields.push({
+      name: fieldName,
+      maxCount: 1
+    })
+  }
+
+  return multer({ storage }).fields(fields)
+}
+
+function createAnyReqFiles (
+  mimeTypes: { [id: string]: string | string[] },
+  fileFilter: (req: express.Request, file: Express.Multer.File, cb: (err: Error, result: boolean) => void) => void
+): RequestHandler {
+  const storage = diskStorage({
+    destination: (req, file, cb) => {
+      cb(null, CONFIG.STORAGE.TMP_DIR)
+    },
+
+    filename: (req, file, cb) => {
+      return generateReqFilename(file, mimeTypes, cb)
+    }
+  })
+
+  return multer({ storage, fileFilter }).any()
+}
+
+function isUserAbleToSearchRemoteURI (res: express.Response) {
+  const user = res.locals.oauth ? res.locals.oauth.token.User : undefined
+
+  return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
+    (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
+}
+
+function getCountVideos (req: express.Request) {
+  return req.query.skipCount !== true
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  buildNSFWFilter,
+  getHostWithPort,
+  createAnyReqFiles,
+  isUserAbleToSearchRemoteURI,
+  createReqFiles,
+  cleanUpReqFiles,
+  getCountVideos
+}
+
+// ---------------------------------------------------------------------------
+
+async function generateReqFilename (
+  file: Express.Multer.File,
+  mimeTypes: { [id: string]: string | string[] },
+  cb: (err: Error, name: string) => void
+) {
+  let extension: string
+  const fileExtension = getLowercaseExtension(file.originalname)
+  const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
+
+  // Take the file extension if we don't understand the mime type
+  if (!extensionFromMimetype) {
+    extension = fileExtension
+  } else {
+    // Take the first available extension for this mimetype
+    extension = extensionFromMimetype
+  }
+
+  let randomString = ''
+
+  try {
+    randomString = await generateRandomString(16)
+  } catch (err) {
+    logger.error('Cannot generate random string for file name.', { err })
+    randomString = 'fake-random-string'
+  }
+
+  cb(null, randomString + extension)
+}

server/core/helpers/ffmpeg/codecs.ts

@@ -0,0 +1,64 @@
+import { FfprobeData } from 'fluent-ffmpeg'
+import { getAudioStream, getVideoStream } from '@peertube/peertube-ffmpeg'
+import { logger } from '../logger.js'
+import { forceNumber } from '@peertube/peertube-core-utils'
+
+export async function getVideoStreamCodec (path: string) {
+  const videoStream = await getVideoStream(path)
+  if (!videoStream) return ''
+
+  const videoCodec = videoStream.codec_tag_string
+
+  if (videoCodec === 'vp09') return 'vp09.00.50.08'
+  if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0'
+
+  const baseProfileMatrix = {
+    avc1: {
+      High: '6400',
+      Main: '4D40',
+      Baseline: '42E0'
+    },
+    av01: {
+      High: '1',
+      Main: '0',
+      Professional: '2'
+    }
+  }
+
+  let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile]
+  if (!baseProfile) {
+    logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
+    baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback
+  }
+
+  if (videoCodec === 'av01') {
+    let level = videoStream.level.toString()
+    if (level.length === 1) level = `0${level}`
+
+    // Guess the tier indicator and bit depth
+    return `${videoCodec}.${baseProfile}.${level}M.08`
+  }
+
+  let level = forceNumber(videoStream.level).toString(16)
+  if (level.length === 1) level = `0${level}`
+
+  // Default, h264 codec
+  return `${videoCodec}.${baseProfile}${level}`
+}
+
+export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
+  const { audioStream } = await getAudioStream(path, existingProbe)
+
+  if (!audioStream) return ''
+
+  const audioCodecName = audioStream.codec_name
+
+  if (audioCodecName === 'opus') return 'opus'
+  if (audioCodecName === 'vorbis') return 'vorbis'
+  if (audioCodecName === 'aac') return 'mp4a.40.2'
+  if (audioCodecName === 'mp3') return 'mp4a.40.34'
+
+  logger.warn('Cannot get audio codec of %s.', path, { audioStream })
+
+  return 'mp4a.40.2' // Fallback
+}

server/core/helpers/ffmpeg/ffmpeg-image.ts

@@ -0,0 +1,14 @@
+import { FFmpegImage } from '@peertube/peertube-ffmpeg'
+import { getFFmpegCommandWrapperOptions } from './ffmpeg-options.js'
+
+export function processGIF (options: Parameters<FFmpegImage['processGIF']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options)
+}
+
+export function generateThumbnailFromVideo (options: Parameters<FFmpegImage['generateThumbnailFromVideo']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options)
+}
+
+export function convertWebPToJPG (options: Parameters<FFmpegImage['convertWebPToJPG']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options)
+}

server/core/helpers/ffmpeg/ffmpeg-options.ts

@@ -0,0 +1,45 @@
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { FFMPEG_NICE } from '@server/initializers/constants.js'
+import { FFmpegCommandWrapperOptions } from '@peertube/peertube-ffmpeg'
+import { AvailableEncoders } from '@peertube/peertube-models'
+
+type CommandType = 'live' | 'vod' | 'thumbnail'
+
+export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions {
+  return {
+    availableEncoders,
+    profile: getProfile(type),
+
+    niceness: FFMPEG_NICE[type.toUpperCase()],
+    tmpDirectory: CONFIG.STORAGE.TMP_DIR,
+    threads: getThreads(type),
+
+    logger: {
+      debug: logger.debug.bind(logger),
+      info: logger.info.bind(logger),
+      warn: logger.warn.bind(logger),
+      error: logger.error.bind(logger)
+    },
+    lTags: { tags: [ 'ffmpeg' ] }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function getThreads (type: CommandType) {
+  if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS
+  if (type === 'vod') return CONFIG.TRANSCODING.THREADS
+
+  // Auto
+  return 0
+}
+
+function getProfile (type: CommandType) {
+  if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE
+  if (type === 'vod') return CONFIG.TRANSCODING.PROFILE
+
+  return undefined
+}

server/core/helpers/ffmpeg/framerate.ts

@@ -0,0 +1,45 @@
+import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants.js'
+
+export function computeOutputFPS (options: {
+  inputFPS: number
+  resolution: number
+}) {
+  const { resolution } = options
+
+  let fps = options.inputFPS
+
+  if (
+    // On small/medium resolutions, limit FPS
+    resolution !== undefined &&
+    resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
+    fps > VIDEO_TRANSCODING_FPS.AVERAGE
+  ) {
+    // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
+    fps = getClosestFramerateStandard({ fps, type: 'STANDARD' })
+  }
+
+  if (fps < VIDEO_TRANSCODING_FPS.HARD_MIN) {
+    throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.HARD_MIN}`)
+  }
+
+  // Cap min FPS
+  if (fps < VIDEO_TRANSCODING_FPS.SOFT_MIN) fps = VIDEO_TRANSCODING_FPS.SOFT_MIN
+  // Cap max FPS
+  if (fps > VIDEO_TRANSCODING_FPS.SOFT_MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' })
+
+  return fps
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function getClosestFramerateStandard (options: {
+  fps: number
+  type: 'HD_STANDARD' | 'STANDARD'
+}) {
+  const { fps, type } = options
+
+  return VIDEO_TRANSCODING_FPS[type].slice(0)
+                                    .sort((a, b) => fps % a - fps % b)[0]
+}

server/core/helpers/ffmpeg/index.ts

@@ -0,0 +1,4 @@
+export * from './codecs.js'
+export * from './ffmpeg-image.js'
+export * from './ffmpeg-options.js'
+export * from './framerate.js'

server/core/helpers/fs.ts

@@ -0,0 +1,12 @@
+import { move } from 'fs-extra/esm'
+import { rename } from 'fs/promises'
+
+export async function tryAtomicMove (src: string, destination: string) {
+  try {
+    await rename(src, destination)
+  } catch (err) {
+    if (err?.code !== 'EXDEV') throw err
+
+    return move(src, destination, { overwrite: true })
+  }
+}

server/core/helpers/geo-ip.ts

@@ -0,0 +1,147 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { pathExists } from 'fs-extra/esm'
+import { writeFile } from 'fs/promises'
+import throttle from 'lodash-es/throttle.js'
+import maxmind, { CityResponse, CountryResponse, Reader } from 'maxmind'
+import { join } from 'path'
+import { isArray } from './custom-validators/misc.js'
+import { logger, loggerTagsFactory } from './logger.js'
+import { isBinaryResponse, peertubeGot } from './requests.js'
+
+const lTags = loggerTagsFactory('geo-ip')
+
+export class GeoIP {
+  private static instance: GeoIP
+
+  private countryReader: Reader<CountryResponse>
+  private cityReader: Reader<CityResponse>
+
+  private readonly INIT_READERS_RETRY_INTERVAL = 1000 * 60 * 10 // 10 minutes
+  private readonly countryDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-country-lite-latest.mmdb')
+  private readonly cityDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-city-lite-latest.mmdb')
+
+  private constructor () {
+  }
+
+  async safeIPISOLookup (ip: string): Promise<{ country: string, subdivisionName: string }> {
+    const emptyResult = { country: null, subdivisionName: null }
+    if (CONFIG.GEO_IP.ENABLED === false) return emptyResult
+
+    try {
+      await this.initReadersIfNeededThrottle()
+
+      const countryResult = this.countryReader?.get(ip)
+      const cityResult = this.cityReader?.get(ip)
+
+      return {
+        country: this.getISOCountry(countryResult),
+        subdivisionName: this.getISOSubdivision(cityResult)
+      }
+    } catch (err) {
+      logger.error('Cannot get country/city information from IP.', { err })
+
+      return emptyResult
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private getISOCountry (countryResult: CountryResponse) {
+    return countryResult?.country?.iso_code || null
+  }
+
+  private getISOSubdivision (subdivisionResult: CityResponse) {
+    const subdivisions = subdivisionResult?.subdivisions
+    if (!isArray(subdivisions) || subdivisions.length === 0) return null
+
+    // The last subdivision is the more precise one
+    const subdivision = subdivisions[subdivisions.length - 1]
+
+    return subdivision.names?.en || null
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async updateDatabases () {
+    if (CONFIG.GEO_IP.ENABLED === false) return
+
+    await this.updateCountryDatabase()
+    await this.updateCityDatabase()
+  }
+
+  private async updateCountryDatabase () {
+    if (!CONFIG.GEO_IP.COUNTRY.DATABASE_URL) return false
+
+    await this.updateDatabaseFile(CONFIG.GEO_IP.COUNTRY.DATABASE_URL, this.countryDBPath)
+
+    this.countryReader = undefined
+
+    return true
+  }
+
+  private async updateCityDatabase () {
+    if (!CONFIG.GEO_IP.CITY.DATABASE_URL) return false
+
+    await this.updateDatabaseFile(CONFIG.GEO_IP.CITY.DATABASE_URL, this.cityDBPath)
+
+    this.cityReader = undefined
+
+    return true
+  }
+
+  private async updateDatabaseFile (url: string, destination: string) {
+    logger.info('Updating GeoIP databases from %s.', url, lTags())
+
+    const gotOptions = { context: { bodyKBLimit: 800_000 }, responseType: 'buffer' as 'buffer' }
+
+    try {
+      const gotResult = await peertubeGot(url, gotOptions)
+
+      if (!isBinaryResponse(gotResult)) {
+        throw new Error('Not a binary response')
+      }
+
+      await writeFile(destination, gotResult.body)
+
+      logger.info('GeoIP database updated %s.', destination, lTags())
+    } catch (err) {
+      logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() })
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async initReadersIfNeeded () {
+    if (!this.countryReader) {
+      let open = true
+
+      if (!await pathExists(this.countryDBPath)) {
+        open = await this.updateCountryDatabase()
+      }
+
+      if (open) {
+        this.countryReader = await maxmind.open(this.countryDBPath)
+      }
+    }
+
+    if (!this.cityReader) {
+      let open = true
+
+      if (!await pathExists(this.cityDBPath)) {
+        open = await this.updateCityDatabase()
+      }
+
+      if (open) {
+        this.cityReader = await maxmind.open(this.cityDBPath)
+      }
+    }
+  }
+
+  private readonly initReadersIfNeededThrottle = throttle(this.initReadersIfNeeded.bind(this), this.INIT_READERS_RETRY_INTERVAL)
+
+  // ---------------------------------------------------------------------------
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/helpers/image-utils.ts

@@ -0,0 +1,155 @@
+import { copy, remove } from 'fs-extra/esm'
+import { readFile, rename } from 'fs/promises'
+import { ColorActionName } from '@jimp/plugin-color'
+import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils'
+import { convertWebPToJPG, processGIF } from './ffmpeg/index.js'
+import { logger } from './logger.js'
+
+import type Jimp from 'jimp'
+
+export function generateImageFilename (extension = '.jpg') {
+  return buildUUID() + extension
+}
+
+export async function processImage (options: {
+  path: string
+  destination: string
+  newSize: { width: number, height: number }
+  keepOriginal?: boolean // default false
+}) {
+  const { path, destination, newSize, keepOriginal = false } = options
+
+  const extension = getLowercaseExtension(path)
+
+  if (path === destination) {
+    throw new Error('Jimp/FFmpeg needs an input path different that the output path.')
+  }
+
+  logger.debug('Processing image %s to %s.', path, destination)
+
+  // Use FFmpeg to process GIF
+  if (extension === '.gif') {
+    await processGIF({ path, destination, newSize })
+  } else {
+    await jimpProcessor({ path, destination, newSize, inputExt: extension })
+  }
+
+  if (keepOriginal !== true) await remove(path)
+
+  logger.debug('Finished processing image %s to %s.', path, destination)
+}
+
+export async function getImageSize (path: string) {
+  const inputBuffer = await readFile(path)
+
+  const Jimp = await import('jimp')
+
+  const image = await Jimp.default.read(inputBuffer)
+
+  return {
+    width: image.getWidth(),
+    height: image.getHeight()
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function jimpProcessor (options: {
+  path: string
+  destination: string
+  newSize: {
+    width: number
+    height: number
+  }
+  inputExt: string
+}) {
+  const { path, destination, newSize, inputExt } = options
+
+  let sourceImage: Jimp
+  const inputBuffer = await readFile(path)
+
+  const Jimp = await import('jimp')
+
+  try {
+    sourceImage = await Jimp.default.read(inputBuffer)
+  } catch (err) {
+    logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err })
+
+    const newName = path + '.jpg'
+    await convertWebPToJPG({ path, destination: newName })
+    await rename(newName, path)
+
+    sourceImage = await Jimp.default.read(path)
+  }
+
+  await remove(destination)
+
+  // Optimization if the source file has the appropriate size
+  const outputExt = getLowercaseExtension(destination)
+  if (skipProcessing({ sourceImage, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt })) {
+    return copy(path, destination)
+  }
+
+  await autoResize({ sourceImage, newSize, destination })
+}
+
+async function autoResize (options: {
+  sourceImage: Jimp
+  newSize: { width: number, height: number }
+  destination: string
+}) {
+  const { sourceImage, newSize, destination } = options
+
+  // Portrait mode targeting a landscape, apply some effect on the image
+  const sourceIsPortrait = sourceImage.getWidth() <= sourceImage.getHeight()
+  const destIsPortraitOrSquare = newSize.width <= newSize.height
+
+  removeExif(sourceImage)
+
+  if (sourceIsPortrait && !destIsPortraitOrSquare) {
+    const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height)
+                                              .color([ { apply: ColorActionName.SHADE, params: [ 50 ] } ])
+
+    const topImage = sourceImage.cloneQuiet().contain(newSize.width, newSize.height)
+
+    return write(baseImage.blit(topImage, 0, 0), destination)
+  }
+
+  return write(sourceImage.cover(newSize.width, newSize.height), destination)
+}
+
+function write (image: Jimp, destination: string) {
+  return image.quality(80).writeAsync(destination)
+}
+
+function skipProcessing (options: {
+  sourceImage: Jimp
+  newSize: { width: number, height: number }
+  imageBytes: number
+  inputExt: string
+  outputExt: string
+}) {
+  const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options
+  const { width, height } = newSize
+
+  if (hasExif(sourceImage)) return false
+  if (sourceImage.getWidth() !== width || sourceImage.getHeight() !== height) return false
+  if (inputExt !== outputExt) return false
+
+  const kB = 1000
+
+  if (height >= 1000) return imageBytes <= 200 * kB
+  if (height >= 500) return imageBytes <= 100 * kB
+
+  return imageBytes <= 15 * kB
+}
+
+function hasExif (image: Jimp) {
+  return !!(image.bitmap as any).exifBuffer
+}
+
+function removeExif (image: Jimp) {
+  (image.bitmap as any).exifBuffer = null
+}

server/core/helpers/logger.ts

@@ -0,0 +1,201 @@
+import { context, trace } from '@opentelemetry/api'
+import { omit } from '@peertube/peertube-core-utils'
+import { stat } from 'fs/promises'
+import { join } from 'path'
+import { format as sqlFormat } from 'sql-formatter'
+import { createLogger, format, transports } from 'winston'
+import { FileTransportOptions } from 'winston/lib/winston/transports'
+import { CONFIG } from '../initializers/config.js'
+import { LOG_FILENAME } from '../initializers/constants.js'
+
+const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
+
+const consoleLoggerFormat = format.printf(info => {
+  let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2)
+
+  if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
+  else additionalInfos = ' ' + additionalInfos
+
+  if (info.sql) {
+    if (CONFIG.LOG.PRETTIFY_SQL) {
+      additionalInfos += '\n' + sqlFormat(info.sql, {
+        language: 'sql',
+        tabWidth: 2
+      })
+    } else {
+      additionalInfos += ' - ' + info.sql
+    }
+  }
+
+  return `[${info.label}] ${info.timestamp} ${info.level}: ${info.message}${additionalInfos}`
+})
+
+const jsonLoggerFormat = format.printf(info => {
+  return JSON.stringify(info, removeCyclicValues())
+})
+
+const timestampFormatter = format.timestamp({
+  format: 'YYYY-MM-DD HH:mm:ss.SSS'
+})
+const labelFormatter = (suffix?: string) => {
+  return format.label({
+    label: suffix ? `${label} ${suffix}` : label
+  })
+}
+
+const fileLoggerOptions: FileTransportOptions = {
+  filename: join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME),
+  handleExceptions: true,
+  format: format.combine(
+    format.timestamp(),
+    jsonLoggerFormat
+  )
+}
+
+if (CONFIG.LOG.ROTATION.ENABLED) {
+  fileLoggerOptions.maxsize = CONFIG.LOG.ROTATION.MAX_FILE_SIZE
+  fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES
+}
+
+function buildLogger (labelSuffix?: string) {
+  return createLogger({
+    level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL,
+    defaultMeta: {
+      get traceId () { return trace.getSpanContext(context.active())?.traceId },
+      get spanId () { return trace.getSpanContext(context.active())?.spanId },
+      get traceFlags () { return trace.getSpanContext(context.active())?.traceFlags }
+    },
+    format: format.combine(
+      labelFormatter(labelSuffix),
+      format.splat()
+    ),
+    transports: [
+      new transports.File(fileLoggerOptions),
+      new transports.Console({
+        handleExceptions: true,
+        format: format.combine(
+          timestampFormatter,
+          format.colorize(),
+          consoleLoggerFormat
+        )
+      })
+    ],
+    exitOnError: true
+  })
+}
+
+const logger = buildLogger()
+
+// ---------------------------------------------------------------------------
+
+function bunyanLogFactory (level: string) {
+  return function (...params: any[]) {
+    let meta = null
+    let args = [].concat(params)
+
+    if (arguments[0] instanceof Error) {
+      meta = arguments[0].toString()
+      args = Array.prototype.slice.call(arguments, 1)
+      args.push(meta)
+    } else if (typeof (args[0]) !== 'string') {
+      meta = arguments[0]
+      args = Array.prototype.slice.call(arguments, 1)
+      args.push(meta)
+    }
+
+    logger[level].apply(logger, args)
+  }
+}
+
+const bunyanLogger = {
+  level: () => { },
+  trace: bunyanLogFactory('debug'),
+  debug: bunyanLogFactory('debug'),
+  verbose: bunyanLogFactory('debug'),
+  info: bunyanLogFactory('info'),
+  warn: bunyanLogFactory('warn'),
+  error: bunyanLogFactory('error'),
+  fatal: bunyanLogFactory('error')
+}
+
+// ---------------------------------------------------------------------------
+
+type LoggerTags = { tags: (string | number)[] }
+type LoggerTagsFn = (...tags: (string | number)[]) => LoggerTags
+function loggerTagsFactory (...defaultTags: (string | number)[]): LoggerTagsFn {
+  return (...tags: (string | number)[]) => {
+    return { tags: defaultTags.concat(tags) }
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+async function mtimeSortFilesDesc (files: string[], basePath: string) {
+  const promises = []
+  const out: { file: string, mtime: number }[] = []
+
+  for (const file of files) {
+    const p = stat(basePath + '/' + file)
+      .then(stats => {
+        if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() })
+      })
+
+    promises.push(p)
+  }
+
+  await Promise.all(promises)
+
+  out.sort((a, b) => b.mtime - a.mtime)
+
+  return out
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+
+  buildLogger, bunyanLogger, consoleLoggerFormat,
+  jsonLoggerFormat, labelFormatter, logger,
+  loggerTagsFactory, mtimeSortFilesDesc, timestampFormatter, type LoggerTags, type LoggerTagsFn
+}
+
+// ---------------------------------------------------------------------------
+
+function removeCyclicValues () {
+  const seen = new WeakSet()
+
+  // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples
+  return (key: string, value: any) => {
+    if (key === 'cert') return 'Replaced by the logger to avoid large log message'
+
+    if (typeof value === 'object' && value !== null) {
+      if (seen.has(value)) return
+
+      seen.add(value)
+    }
+
+    if (value instanceof Set) {
+      return Array.from(value)
+    }
+
+    if (value instanceof Map) {
+      return Array.from(value.entries())
+    }
+
+    if (value instanceof Error) {
+      const error = {}
+
+      Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
+
+      return error
+    }
+
+    return value
+  }
+}
+
+function getAdditionalInfo (info: any) {
+  const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
+
+  return omit(info, toOmit)
+}

server/core/helpers/markdown.ts

@@ -0,0 +1,91 @@
+import MarkdownItClass from 'markdown-it'
+// FIXME: use direct import: import markdownItEmoji from 'markdown-it-emoji/lib/light.mjs' if it improves perf'
+// when https://github.com/privatenumber/tsx/issues/334 is fixed
+import { light as markdownItEmoji } from 'markdown-it-emoji'
+import sanitizeHtml from 'sanitize-html'
+import { getDefaultSanitizeOptions, getTextOnlySanitizeOptions, TEXT_WITH_HTML_RULES } from '@peertube/peertube-core-utils'
+
+const defaultSanitizeOptions = getDefaultSanitizeOptions()
+const textOnlySanitizeOptions = getTextOnlySanitizeOptions()
+
+const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true })
+  .enable(TEXT_WITH_HTML_RULES)
+  .use(markdownItEmoji)
+
+const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false })
+  .use(markdownItEmoji)
+  .use(plainTextPlugin)
+
+const toSafeHtml = (text: string) => {
+  if (!text) return ''
+
+  // Restore line feed
+  const textWithLineFeed = text.replace(/<br.?\/?>/g, '\r\n')
+
+  // Convert possible markdown (emojis, emphasis and lists) to html
+  const html = markdownItForSafeHtml.render(textWithLineFeed)
+
+  // Convert to safe Html
+  return sanitizeHtml(html, defaultSanitizeOptions)
+}
+
+const mdToOneLinePlainText = (text: string) => {
+  if (!text) return ''
+
+  markdownItForPlainText.render(text)
+
+  // Convert to safe Html
+  return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  toSafeHtml,
+  mdToOneLinePlainText
+}
+
+// ---------------------------------------------------------------------------
+
+// Thanks: https://github.com/wavesheep/markdown-it-plain-text
+function plainTextPlugin (markdownIt: any) {
+  function plainTextRule (state: any) {
+    const text = scan(state.tokens)
+
+    markdownIt.plainText = text
+  }
+
+  function scan (tokens: any[]) {
+    let lastSeparator = ''
+    let text = ''
+
+    function buildSeparator (token: any) {
+      if (token.type === 'list_item_close') {
+        lastSeparator = ', '
+      }
+
+      if (token.tag === 'br' || token.type === 'paragraph_close') {
+        lastSeparator = ' '
+      }
+    }
+
+    for (const token of tokens) {
+      buildSeparator(token)
+
+      if (token.type !== 'inline') continue
+
+      for (const child of token.children) {
+        buildSeparator(child)
+
+        if (!child.content) continue
+
+        text += lastSeparator + child.content
+        lastSeparator = ''
+      }
+    }
+
+    return text
+  }
+
+  markdownIt.core.ruler.push('plainText', plainTextRule)
+}

server/core/helpers/memoize.ts

@@ -0,0 +1,12 @@
+import memoizee from 'memoizee'
+
+export function Memoize (config?: memoizee.Options<any>) {
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const oldFunction = descriptor.value
+    const newFunction = memoizee(oldFunction, config)
+
+    descriptor.value = function () {
+      return newFunction.apply(this, arguments)
+    }
+  }
+}

server/core/helpers/mentions.ts

@@ -0,0 +1,42 @@
+import { uniqify } from '@peertube/peertube-core-utils'
+import { WEBSERVER } from '@server/initializers/constants.js'
+import { actorNameAlphabet } from './custom-validators/activitypub/actor.js'
+import { regexpCapture } from './regexp.js'
+
+export function extractMentions (text: string, isOwned: boolean) {
+  let result: string[] = []
+
+  const localMention = `@(${actorNameAlphabet}+)`
+  const remoteMention = `${localMention}@${WEBSERVER.HOST}`
+
+  const mentionRegex = isOwned
+    ? '(?:(?:' + remoteMention + ')|(?:' + localMention + '))' // Include local mentions?
+    : '(?:' + remoteMention + ')'
+
+  const firstMentionRegex = new RegExp(`^${mentionRegex} `, 'g')
+  const endMentionRegex = new RegExp(` ${mentionRegex}$`, 'g')
+  const remoteMentionsRegex = new RegExp(' ' + remoteMention + ' ', 'g')
+
+  result = result.concat(
+    regexpCapture(text, firstMentionRegex)
+      .map(([ , username1, username2 ]) => username1 || username2),
+
+    regexpCapture(text, endMentionRegex)
+      .map(([ , username1, username2 ]) => username1 || username2),
+
+    regexpCapture(text, remoteMentionsRegex)
+      .map(([ , username ]) => username)
+  )
+
+  // Include local mentions
+  if (isOwned) {
+    const localMentionsRegex = new RegExp(' ' + localMention + ' ', 'g')
+
+    result = result.concat(
+      regexpCapture(text, localMentionsRegex)
+        .map(([ , username ]) => username)
+    )
+  }
+
+  return uniqify(result)
+}

server/core/helpers/otp.ts

@@ -0,0 +1,58 @@
+import { Secret, TOTP } from 'otpauth'
+import { CONFIG } from '@server/initializers/config.js'
+import { WEBSERVER } from '@server/initializers/constants.js'
+import { decrypt } from './peertube-crypto.js'
+
+async function isOTPValid (options: {
+  encryptedSecret: string
+  token: string
+}) {
+  const { token, encryptedSecret } = options
+
+  const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE)
+
+  const totp = new TOTP({
+    ...baseOTPOptions(),
+
+    secret
+  })
+
+  const delta = totp.validate({
+    token,
+    window: 1
+  })
+
+  if (delta === null) return false
+
+  return true
+}
+
+function generateOTPSecret (email: string) {
+  const totp = new TOTP({
+    ...baseOTPOptions(),
+
+    label: email,
+    secret: new Secret()
+  })
+
+  return {
+    secret: totp.secret.base32,
+    uri: totp.toString()
+  }
+}
+
+export {
+  isOTPValid,
+  generateOTPSecret
+}
+
+// ---------------------------------------------------------------------------
+
+function baseOTPOptions () {
+  return {
+    issuer: WEBSERVER.HOST,
+    algorithm: 'SHA1',
+    digits: 6,
+    period: 30
+  }
+}

server/core/helpers/peertube-crypto.ts

@@ -0,0 +1,116 @@
+import httpSignature from '@peertube/http-signature'
+import { sha256 } from '@peertube/peertube-node-utils'
+import { createCipheriv, createDecipheriv } from 'crypto'
+import { Request } from 'express'
+import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants.js'
+import { MActor } from '../types/models/index.js'
+import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils.js'
+import { logger } from './logger.js'
+
+function createPrivateAndPublicKeys () {
+  logger.info('Generating a RSA key...')
+
+  return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE)
+}
+
+// ---------------------------------------------------------------------------
+// User password checks
+// ---------------------------------------------------------------------------
+
+async function comparePassword (plainPassword: string, hashPassword: string) {
+  if (!plainPassword) return false
+
+  const { compare } = await import('bcrypt')
+
+  return compare(plainPassword, hashPassword)
+}
+
+async function cryptPassword (password: string) {
+  const { genSalt, hash } = await import('bcrypt')
+
+  const salt = await genSalt(BCRYPT_SALT_SIZE)
+
+  return hash(password, salt)
+}
+
+// ---------------------------------------------------------------------------
+// HTTP Signature
+// ---------------------------------------------------------------------------
+
+function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean {
+  if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) {
+    return buildDigest(rawBody.toString()) === req.headers['digest']
+  }
+
+  return true
+}
+
+function isHTTPSignatureVerified (httpSignatureParsed: any, actor: MActor): boolean {
+  return httpSignature.verifySignature(httpSignatureParsed, actor.publicKey) === true
+}
+
+function parseHTTPSignature (req: Request, clockSkew?: number) {
+  const requiredHeaders = req.method === 'POST'
+    ? [ '(request-target)', 'host', 'digest' ]
+    : [ '(request-target)', 'host' ]
+
+  const parsed = httpSignature.parse(req, { clockSkew, headers: requiredHeaders })
+
+  const parsedHeaders = parsed.params.headers
+  if (!parsedHeaders.includes('date') && !parsedHeaders.includes('(created)')) {
+    throw new Error(`date or (created) must be included in signature`)
+  }
+
+  return parsed
+}
+
+// ---------------------------------------------------------------------------
+
+function buildDigest (body: any) {
+  const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
+
+  return 'SHA-256=' + sha256(rawBody, 'base64')
+}
+
+// ---------------------------------------------------------------------------
+// Encryption
+// ---------------------------------------------------------------------------
+
+async function encrypt (str: string, secret: string) {
+  const iv = await randomBytesPromise(ENCRYPTION.IV)
+
+  const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
+  const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv)
+
+  let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':'
+  encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING)
+  encrypted += cipher.final(ENCRYPTION.ENCODING)
+
+  return encrypted
+}
+
+async function decrypt (encryptedArg: string, secret: string) {
+  const [ ivStr, encryptedStr ] = encryptedArg.split(':')
+
+  const iv = Buffer.from(ivStr, 'hex')
+  const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
+
+  const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv)
+
+  return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8')
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isHTTPSignatureDigestValid,
+  parseHTTPSignature,
+  isHTTPSignatureVerified,
+  buildDigest,
+  comparePassword,
+  createPrivateAndPublicKeys,
+  cryptPassword,
+
+  encrypt,
+  decrypt
+}

server/core/helpers/peertube-jsonld.ts

@@ -0,0 +1,162 @@
+import { omit } from '@peertube/peertube-core-utils'
+import { sha256 } from '@peertube/peertube-node-utils'
+import { createSign, createVerify } from 'crypto'
+import cloneDeep from 'lodash-es/cloneDeep.js'
+import { MActor } from '../types/models/index.js'
+import { getAllContext } from './activity-pub-utils.js'
+import { jsonld } from './custom-jsonld-signature.js'
+import { isArray } from './custom-validators/misc.js'
+import { logger } from './logger.js'
+import { assertIsInWorkerThread } from './threads.js'
+
+type ExpressRequest = { body: any }
+
+export function compactJSONLDAndCheckSignature (fromActor: MActor, req: ExpressRequest): Promise<boolean> {
+  if (req.body.signature.type === 'RsaSignature2017') {
+    return compactJSONLDAndCheckRSA2017Signature(fromActor, req)
+  }
+
+  logger.warn('Unknown JSON LD signature %s.', req.body.signature.type, req.body)
+
+  return Promise.resolve(false)
+}
+
+// Backward compatibility with "other" implementations
+export async function compactJSONLDAndCheckRSA2017Signature (fromActor: MActor, req: ExpressRequest) {
+  const compacted = await jsonldCompact(omit(req.body, [ 'signature' ]))
+
+  fixCompacted(req.body, compacted)
+
+  req.body = { ...compacted, signature: req.body.signature }
+
+  if (compacted['@include']) {
+    logger.warn('JSON-LD @include is not supported')
+    return false
+  }
+
+  // TODO: compat with < 6.1, remove in 7.0
+  let safe = true
+  if (
+    (compacted.type === 'Create' && (compacted?.object?.type === 'WatchAction' || compacted?.object?.type === 'CacheFile')) ||
+    (compacted.type === 'Undo' && compacted?.object?.type === 'Create' && compacted?.object?.object.type === 'CacheFile')
+  ) {
+    safe = false
+  }
+
+  const [ documentHash, optionsHash ] = await Promise.all([
+    hashObject(compacted, safe),
+    createSignatureHash(req.body.signature, safe)
+  ])
+
+  const toVerify = optionsHash + documentHash
+
+  const verify = createVerify('RSA-SHA256')
+  verify.update(toVerify, 'utf8')
+
+  return verify.verify(fromActor.publicKey, req.body.signature.signatureValue, 'base64')
+}
+
+function fixCompacted (original: any, compacted: any) {
+  if (!original || !compacted) return
+
+  for (const [ k, v ] of Object.entries(original)) {
+    if (k === '@context' || k === 'signature') continue
+    if (v === undefined || v === null) continue
+
+    const cv = compacted[k]
+    if (cv === undefined || cv === null) continue
+
+    if (typeof v === 'string') {
+      if (v === 'https://www.w3.org/ns/activitystreams#Public' && cv === 'as:Public') {
+        compacted[k] = v
+      }
+    }
+
+    if (isArray(v) && !isArray(cv)) {
+      compacted[k] = [ cv ]
+
+      for (let i = 0; i < v.length; i++) {
+        if (v[i] === 'https://www.w3.org/ns/activitystreams#Public' && cv[i] === 'as:Public') {
+          compacted[k][i] = v[i]
+        }
+      }
+    }
+
+    if (typeof v === 'object') {
+      fixCompacted(original[k], compacted[k])
+    }
+  }
+}
+
+export async function signJsonLDObject <T> (options: {
+  byActor: { url: string, privateKey: string }
+  data: T
+  disableWorkerThreadAssertion?: boolean
+}) {
+  const { byActor, data, disableWorkerThreadAssertion = false } = options
+
+  if (!disableWorkerThreadAssertion) assertIsInWorkerThread()
+
+  const signature = {
+    type: 'RsaSignature2017',
+    creator: byActor.url,
+    created: new Date().toISOString()
+  }
+
+  const [ documentHash, optionsHash ] = await Promise.all([
+    createDocWithoutSignatureHash(data),
+    createSignatureHash(signature)
+  ])
+
+  const toSign = optionsHash + documentHash
+
+  const sign = createSign('RSA-SHA256')
+  sign.update(toSign, 'utf8')
+
+  const signatureValue = sign.sign(byActor.privateKey, 'base64')
+  Object.assign(signature, { signatureValue })
+
+  return Object.assign(data, { signature })
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function hashObject (obj: any, safe: boolean): Promise<any> {
+  const res = await jsonldNormalize(obj, safe)
+
+  return sha256(res)
+}
+
+function jsonldCompact (obj: any) {
+  return (jsonld as any).promises.compact(obj, getAllContext())
+}
+
+function jsonldNormalize (obj: any, safe: boolean) {
+  return (jsonld as any).promises.normalize(obj, {
+    safe,
+    algorithm: 'URDNA2015',
+    format: 'application/n-quads'
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function createSignatureHash (signature: any, safe = true) {
+  return hashObject({
+    '@context': [
+      'https://w3id.org/security/v1',
+      { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' }
+    ],
+
+    ...omit(signature, [ 'type', 'id', 'signatureValue' ])
+  }, safe)
+}
+
+function createDocWithoutSignatureHash (doc: any) {
+  const docWithoutSignature = cloneDeep(doc)
+  delete docWithoutSignature.signature
+
+  return hashObject(docWithoutSignature, true)
+}

server/core/helpers/promise-cache.ts

@@ -0,0 +1,39 @@
+export class CachePromiseFactory <A, R> {
+  private readonly running = new Map<string, Promise<R>>()
+
+  constructor (
+    private readonly fn: (arg: A) => Promise<R>,
+    private readonly keyBuilder: (arg: A) => string
+  ) {
+  }
+
+  run (arg: A) {
+    return this.runWithContext(null, arg)
+  }
+
+  runWithContext (ctx: any, arg: A) {
+    const key = this.keyBuilder(arg)
+
+    if (this.running.has(key)) return this.running.get(key)
+
+    const p = this.fn.apply(ctx || this, [ arg ])
+
+    this.running.set(key, p)
+
+    return p.finally(() => this.running.delete(key))
+  }
+}
+
+export function CachePromise (options: {
+  keyBuilder: (...args: any[]) => string
+}) {
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const promiseCache = new CachePromiseFactory(descriptor.value, options.keyBuilder)
+
+    descriptor.value = function () {
+      if (arguments.length !== 1) throw new Error('Cache promise only support methods with 1 argument')
+
+      return promiseCache.runWithContext(this, arguments[0])
+    }
+  }
+}

server/core/helpers/proxy.ts

@@ -0,0 +1,14 @@
+function getProxy () {
+  return process.env.HTTPS_PROXY ||
+         process.env.HTTP_PROXY ||
+         undefined
+}
+
+function isProxyEnabled () {
+  return !!getProxy()
+}
+
+export {
+  getProxy,
+  isProxyEnabled
+}

server/core/helpers/query.ts

@@ -0,0 +1,81 @@
+import { pick } from '@peertube/peertube-core-utils'
+import {
+  VideoChannelsSearchQueryAfterSanitize,
+  VideoPlaylistsSearchQueryAfterSanitize,
+  VideosCommonQueryAfterSanitize,
+  VideosSearchQueryAfterSanitize
+} from '@peertube/peertube-models'
+
+function pickCommonVideoQuery (query: VideosCommonQueryAfterSanitize) {
+  return pick(query, [
+    'start',
+    'count',
+    'sort',
+    'nsfw',
+    'isLive',
+    'categoryOneOf',
+    'licenceOneOf',
+    'languageOneOf',
+    'privacyOneOf',
+    'tagsOneOf',
+    'tagsAllOf',
+    'isLocal',
+    'include',
+    'skipCount',
+    'hasHLSFiles',
+    'hasWebtorrentFiles', // TODO: Remove in v7
+    'hasWebVideoFiles',
+    'search',
+    'excludeAlreadyWatched',
+    'autoTagOneOf'
+  ])
+}
+
+function pickSearchVideoQuery (query: VideosSearchQueryAfterSanitize) {
+  return {
+    ...pickCommonVideoQuery(query),
+
+    ...pick(query, [
+      'searchTarget',
+      'host',
+      'startDate',
+      'endDate',
+      'originallyPublishedStartDate',
+      'originallyPublishedEndDate',
+      'durationMin',
+      'durationMax',
+      'uuids'
+    ])
+  }
+}
+
+function pickSearchChannelQuery (query: VideoChannelsSearchQueryAfterSanitize) {
+  return pick(query, [
+    'searchTarget',
+    'search',
+    'start',
+    'count',
+    'sort',
+    'host',
+    'handles'
+  ])
+}
+
+function pickSearchPlaylistQuery (query: VideoPlaylistsSearchQueryAfterSanitize) {
+  return pick(query, [
+    'searchTarget',
+    'search',
+    'start',
+    'count',
+    'sort',
+    'host',
+    'uuids'
+  ])
+}
+
+export {
+  pickCommonVideoQuery,
+  pickSearchVideoQuery,
+  pickSearchPlaylistQuery,
+  pickSearchChannelQuery
+}

server/core/helpers/regexp.ts

@@ -0,0 +1,32 @@
+// Thanks to https://regex101.com
+export function regexpCapture (str: string, regex: RegExp, maxIterations = 100) {
+  const result: RegExpExecArray[] = []
+  let m: RegExpExecArray
+  let i = 0
+
+  while ((m = regex.exec(str)) !== null && i < maxIterations) {
+    // This is necessary to avoid infinite loops with zero-width matches
+    if (m.index === regex.lastIndex) {
+      regex.lastIndex++
+    }
+
+    result.push(m)
+    i++
+  }
+
+  return result
+}
+
+export function wordsToRegExp (words: string[]) {
+  if (words.length === 0) throw new Error('Need words with at least one element')
+
+  const innerRegex = words
+    .map(word => escapeForRegex(word.trim()))
+    .join('|')
+
+  return new RegExp(`(?:\\P{L}|^)(?:${innerRegex})(?=\\P{L}|$)`, 'iu')
+}
+
+export function escapeForRegex (value: string) {
+  return value.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')
+}

server/core/helpers/requests.ts

@@ -0,0 +1,258 @@
+import httpSignature from '@peertube/http-signature'
+import { createWriteStream } from 'fs'
+import { remove } from 'fs-extra/esm'
+import got, { CancelableRequest, OptionsInit, OptionsOfTextResponseBody, OptionsOfUnknownResponseBody, RequestError, Response } from 'got'
+import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
+import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants.js'
+import { pipelinePromise } from './core-utils.js'
+import { logger, loggerTagsFactory } from './logger.js'
+import { getProxy, isProxyEnabled } from './proxy.js'
+
+const lTags = loggerTagsFactory('request')
+
+export interface PeerTubeRequestError extends Error {
+  statusCode?: number
+  responseBody?: any
+  responseHeaders?: any
+  requestHeaders?: any
+}
+
+type PeerTubeRequestOptions = {
+  timeout?: number
+  activityPub?: boolean
+  bodyKBLimit?: number // 1MB
+
+  httpSignature?: {
+    algorithm: string
+    authorizationHeaderName: string
+    keyId: string
+    key: string
+    headers: string[]
+  }
+
+  jsonResponse?: boolean
+
+  followRedirect?: boolean
+} & Pick<OptionsInit, 'headers' | 'json' | 'method' | 'searchParams'>
+
+const peertubeGot = got.extend({
+  ...getAgent(),
+
+  headers: {
+    'user-agent': getUserAgent()
+  },
+
+  handlers: [
+    (options, next) => {
+      const promiseOrStream = next(options) as CancelableRequest<any>
+      const bodyKBLimit = options.context?.bodyKBLimit as number
+      if (!bodyKBLimit) throw new Error('No KB limit for this request')
+
+      const bodyLimit = bodyKBLimit * 1000
+
+      /* eslint-disable @typescript-eslint/no-floating-promises */
+      promiseOrStream.on('downloadProgress', progress => {
+        if (progress.transferred > bodyLimit && progress.percent !== 1) {
+          const message = `Exceeded the download limit of ${bodyLimit} B`
+          logger.warn(message, lTags())
+
+          // CancelableRequest
+          if (promiseOrStream.cancel) {
+            promiseOrStream.cancel()
+            return
+          }
+
+          // Stream
+          (promiseOrStream as any).destroy()
+        }
+      })
+
+      return promiseOrStream
+    }
+  ],
+
+  hooks: {
+    beforeRequest: [
+      options => {
+        const headers = options.headers || {}
+        headers['host'] = buildUrl(options.url).host
+      },
+
+      options => {
+        const httpSignatureOptions = options.context?.httpSignature
+
+        if (httpSignatureOptions) {
+          const method = options.method ?? 'GET'
+          const path = buildUrl(options.url).pathname
+
+          if (!method || !path) {
+            throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
+          }
+
+          httpSignature.signRequest({
+            getHeader: function (header: string) {
+              const value = options.headers[header.toLowerCase()]
+
+              if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header })
+              return value
+            },
+
+            setHeader: function (header: string, value: string) {
+              options.headers[header] = value
+            },
+
+            method,
+            path
+          }, httpSignatureOptions)
+        }
+      }
+    ],
+
+    beforeRetry: [
+      (error: RequestError, retryCount: number) => {
+        logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
+      }
+    ]
+  }
+})
+
+function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions(options) as OptionsOfTextResponseBody
+
+  return peertubeGot(url, gotOptions)
+    .catch(err => { throw buildRequestError(err) })
+}
+
+function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions(options)
+
+  return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
+    .catch(err => { throw buildRequestError(err) })
+}
+
+async function doRequestAndSaveToFile (
+  url: string,
+  destPath: string,
+  options: PeerTubeRequestOptions = {}
+) {
+  const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
+
+  const outFile = createWriteStream(destPath)
+
+  try {
+    await pipelinePromise(
+      peertubeGot.stream(url, { ...gotOptions, isStream: true }),
+      outFile
+    )
+  } catch (err) {
+    remove(destPath)
+      .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
+
+    throw buildRequestError(err)
+  }
+}
+
+function getAgent () {
+  if (!isProxyEnabled()) return {}
+
+  const proxy = getProxy()
+
+  logger.info('Using proxy %s.', proxy, lTags())
+
+  const proxyAgentOptions = {
+    keepAlive: true,
+    keepAliveMsecs: 1000,
+    maxSockets: 256,
+    maxFreeSockets: 256,
+    scheduling: 'lifo' as 'lifo',
+    proxy
+  }
+
+  return {
+    agent: {
+      http: new HttpProxyAgent(proxyAgentOptions),
+      https: new HttpsProxyAgent(proxyAgentOptions)
+    }
+  }
+}
+
+function getUserAgent () {
+  return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
+}
+
+function isBinaryResponse (result: Response<any>) {
+  return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  type PeerTubeRequestOptions,
+
+  doRequest,
+  doJSONRequest,
+  doRequestAndSaveToFile,
+  isBinaryResponse,
+  getAgent,
+  peertubeGot
+}
+
+// ---------------------------------------------------------------------------
+
+function buildGotOptions (options: PeerTubeRequestOptions): OptionsOfUnknownResponseBody {
+  const { activityPub, bodyKBLimit = 1000 } = options
+
+  const context = { bodyKBLimit, httpSignature: options.httpSignature }
+
+  let headers = options.headers || {}
+
+  if (!headers.date) {
+    headers = { ...headers, date: new Date().toUTCString() }
+  }
+
+  if (activityPub && !headers.accept) {
+    headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
+  }
+
+  return {
+    method: options.method,
+    dnsCache: true,
+    timeout: {
+      request: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT
+    },
+    json: options.json,
+    searchParams: options.searchParams,
+    followRedirect: options.followRedirect,
+    retry: {
+      limit: 2
+    },
+    headers,
+    context
+  }
+}
+
+function buildRequestError (error: RequestError) {
+  const newError: PeerTubeRequestError = new Error(error.message)
+  newError.name = error.name
+  newError.stack = error.stack
+
+  if (error.response) {
+    newError.responseBody = error.response.body
+    newError.responseHeaders = error.response.headers
+    newError.statusCode = error.response.statusCode
+  }
+
+  if (error.options) {
+    newError.requestHeaders = error.options.headers
+  }
+
+  return newError
+}
+
+function buildUrl (url: string | URL) {
+  if (typeof url === 'string') {
+    return new URL(url)
+  }
+
+  return url
+}

server/core/helpers/stream-replacer.ts

@@ -0,0 +1,58 @@
+import { Transform, TransformCallback } from 'stream'
+
+// Thanks: https://stackoverflow.com/a/45126242
+class StreamReplacer extends Transform {
+  private pendingChunk: Buffer
+
+  constructor (private readonly replacer: (line: string) => string) {
+    super()
+  }
+
+  _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) {
+    try {
+      this.pendingChunk = this.pendingChunk?.length
+        ? Buffer.concat([ this.pendingChunk, chunk ])
+        : chunk
+
+      let index: number
+
+      // As long as we keep finding newlines, keep making slices of the buffer and push them to the
+      // readable side of the transform stream
+      while ((index = this.pendingChunk.indexOf('\n')) !== -1) {
+        // The `end` parameter is non-inclusive, so increase it to include the newline we found
+        const line = this.pendingChunk.slice(0, ++index)
+
+        // `start` is inclusive, but we are already one char ahead of the newline -> all good
+        this.pendingChunk = this.pendingChunk.slice(index)
+
+        // We have a single line here! Prepend the string we want
+        this.push(this.doReplace(line))
+      }
+
+      return done()
+    } catch (err) {
+      return done(err)
+    }
+  }
+
+  _flush (done: TransformCallback) {
+    // If we have any remaining data in the cache, send it out
+    if (!this.pendingChunk?.length) return done()
+
+    try {
+      return done(null, this.doReplace(this.pendingChunk))
+    } catch (err) {
+      return done(err)
+    }
+  }
+
+  private doReplace (buffer: Buffer) {
+    const line = this.replacer(buffer.toString('utf8'))
+
+    return Buffer.from(line, 'utf8')
+  }
+}
+
+export {
+  StreamReplacer
+}

server/core/helpers/threads.ts

@@ -0,0 +1,8 @@
+import { isMainThread } from 'node:worker_threads'
+import { logger } from './logger.js'
+
+export function assertIsInWorkerThread () {
+  if (!isMainThread) return
+
+  logger.error('Caller is not in worker thread', { stack: new Error().stack })
+}

server/core/helpers/token-generator.ts

@@ -0,0 +1,19 @@
+import { buildUUID } from '@peertube/peertube-node-utils'
+
+function generateRunnerRegistrationToken () {
+  return 'ptrrt-' + buildUUID()
+}
+
+function generateRunnerToken () {
+  return 'ptrt-' + buildUUID()
+}
+
+function generateRunnerJobToken () {
+  return 'ptrjt-' + buildUUID()
+}
+
+export {
+  generateRunnerRegistrationToken,
+  generateRunnerToken,
+  generateRunnerJobToken
+}

server/core/helpers/unzip.ts

@@ -0,0 +1,55 @@
+import { createWriteStream } from 'fs'
+import { ensureDir } from 'fs-extra/esm'
+import { dirname, join } from 'path'
+import { pipeline } from 'stream'
+import * as yauzl from 'yauzl'
+import { logger, loggerTagsFactory } from './logger.js'
+
+const lTags = loggerTagsFactory('unzip')
+
+export async function unzip (source: string, destination: string) {
+  await ensureDir(destination)
+
+  logger.info(`Unzip ${source} to ${destination}`, lTags())
+
+  return new Promise<void>((res, rej) => {
+    yauzl.open(source, { lazyEntries: true }, (err, zipFile) => {
+      if (err) return rej(err)
+
+      zipFile.readEntry()
+
+      zipFile.on('entry', async entry => {
+        const entryPath = join(destination, entry.fileName)
+
+        try {
+          if (/\/$/.test(entry.fileName)) {
+            await ensureDir(entryPath)
+            logger.debug(`Creating directory from zip ${entryPath}`, lTags())
+
+            zipFile.readEntry()
+            return
+          }
+
+          await ensureDir(dirname(entryPath))
+        } catch (err) {
+          return rej(err)
+        }
+
+        zipFile.openReadStream(entry, (readErr, readStream) => {
+          if (readErr) return rej(readErr)
+
+          logger.debug(`Creating file from zip ${entryPath}`, lTags())
+
+          const writeStream = createWriteStream(entryPath)
+          writeStream.on('close', () => zipFile.readEntry())
+
+          pipeline(readStream, writeStream, pipelineErr => {
+            if (pipelineErr) return rej(pipelineErr)
+          })
+        })
+      })
+
+      zipFile.on('end', () => res())
+    })
+  })
+}

server/core/helpers/upload.ts

@@ -0,0 +1,14 @@
+import { join } from 'path'
+import { DIRECTORIES } from '@server/initializers/constants.js'
+
+function getResumableUploadPath (filename?: string) {
+  if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
+
+  return DIRECTORIES.RESUMABLE_UPLOAD
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getResumableUploadPath
+}

server/core/helpers/utils.ts

@@ -0,0 +1,70 @@
+import { remove } from 'fs-extra/esm'
+import { Instance as ParseTorrent } from 'parse-torrent'
+import { join } from 'path'
+import { sha256 } from '@peertube/peertube-node-utils'
+import { ResultList } from '@peertube/peertube-models'
+import { CONFIG } from '../initializers/config.js'
+import { randomBytesPromise } from './core-utils.js'
+import { logger } from './logger.js'
+
+function deleteFileAndCatch (path: string) {
+  remove(path)
+    .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
+}
+
+async function generateRandomString (size: number) {
+  const raw = await randomBytesPromise(size)
+
+  return raw.toString('hex')
+}
+
+interface FormattableToJSON<U, V> {
+  toFormattedJSON (args?: U): V
+}
+
+function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) {
+  const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg))
+
+  return {
+    total: objectsTotal,
+    data: formattedObjects
+  } as ResultList<V>
+}
+
+function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') {
+  const id = typeof target === 'string'
+    ? target
+    : target.infoHash
+
+  const hash = sha256(id)
+  return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
+}
+
+function getSecureTorrentName (originalName: string) {
+  return sha256(originalName) + '.torrent'
+}
+
+/**
+ * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
+ * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
+ * not contain a UUID, returns null.
+ */
+function getUUIDFromFilename (filename: string) {
+  const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
+  const result = filename.match(regex)
+
+  if (!result || Array.isArray(result) === false) return null
+
+  return result[0]
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  deleteFileAndCatch,
+  generateRandomString,
+  getFormattedObjects,
+  getSecureTorrentName,
+  generateVideoImportTmpPath,
+  getUUIDFromFilename
+}

server/core/helpers/version.ts

@@ -0,0 +1,36 @@
+import { execPromise, execPromise2 } from './core-utils.js'
+import { logger } from './logger.js'
+
+async function getServerCommit () {
+  try {
+    const tag = await execPromise2(
+      '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
+      { stdio: [ 0, 1, 2 ] }
+    )
+
+    if (tag) return tag.replace(/^v/, '')
+  } catch (err) {
+    logger.debug('Cannot get version from git tags.', { err })
+  }
+
+  try {
+    const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
+
+    if (version) return version.toString().trim()
+  } catch (err) {
+    logger.debug('Cannot get version from git HEAD.', { err })
+  }
+
+  return ''
+}
+
+function getNodeABIVersion () {
+  const version = process.versions.modules
+
+  return parseInt(version)
+}
+
+export {
+  getServerCommit,
+  getNodeABIVersion
+}

server/core/helpers/video.ts

@@ -0,0 +1,52 @@
+import { VideoPrivacy } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js'
+import { Response } from 'express'
+
+export function getVideoWithAttributes (res: Response) {
+  return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo
+}
+
+export function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
+  return isStreamingPlaylist(videoOrPlaylist)
+    ? videoOrPlaylist.Video
+    : videoOrPlaylist
+}
+
+export function getPrivaciesForFederation () {
+  return (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true)
+    ? [ { privacy: VideoPrivacy.PUBLIC }, { privacy: VideoPrivacy.UNLISTED } ]
+    : [ { privacy: VideoPrivacy.PUBLIC } ]
+}
+
+export function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mimeType: string) {
+  const value = mimeTypes[mimeType]
+
+  if (Array.isArray(value)) return value[0]
+
+  return value
+}
+
+export function peertubeLicenceToSPDX (licence: number) {
+  return {
+    1: 'CC-BY-4.0',
+    2: 'CC-BY-SA-4.0',
+    3: 'CC-BY-ND-4.0',
+    4: 'CC-BY-NC-4.0',
+    5: 'CC-BY-NC-SA-4.0',
+    6: 'CC-BY-NC-ND-4.0',
+    7: 'CC0'
+  }[licence]
+}
+
+export function spdxToPeertubeLicence (licence: string) {
+  return {
+    'CC-BY-4.0': 1,
+    'CC-BY-SA-4.0': 2,
+    'CC-BY-ND-4.0': 3,
+    'CC-BY-NC-4.0': 4,
+    'CC-BY-NC-SA-4.0': 5,
+    'CC-BY-NC-ND-4.0': 6,
+    'CC0': 7
+  }[licence]
+}

server/core/helpers/webtorrent.ts

@@ -0,0 +1,275 @@
+import bencode from 'bencode'
+import createTorrent from 'create-torrent'
+import { createWriteStream } from 'fs'
+import { ensureDir, pathExists, remove } from 'fs-extra/esm'
+import { readFile, writeFile } from 'fs/promises'
+import { encode as magnetUriEncode } from 'magnet-uri'
+import parseTorrent from 'parse-torrent'
+import { dirname, join } from 'path'
+import { pipeline } from 'stream'
+import { promisify2 } from '@peertube/peertube-core-utils'
+import { isArray } from '@server/helpers/custom-validators/misc.js'
+import { WEBSERVER } from '@server/initializers/constants.js'
+import { generateTorrentFileName } from '@server/lib/paths.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file.js'
+import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js'
+import { MVideo } from '@server/types/models/video/video.js'
+import { sha1 } from '@peertube/peertube-node-utils'
+import { CONFIG } from '../initializers/config.js'
+import { logger } from './logger.js'
+import { generateVideoImportTmpPath } from './utils.js'
+import { extractVideo } from './video.js'
+
+import type { Instance, TorrentFile } from 'webtorrent'
+
+const createTorrentPromise = promisify2<string, any, any>(createTorrent)
+
+async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {
+  const id = target.uri || target.torrentName
+  let timer
+
+  const path = generateVideoImportTmpPath(id)
+  logger.info('Importing torrent video %s', id)
+
+  const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')
+  await ensureDir(directoryPath)
+
+  // eslint-disable-next-line new-cap
+  const webtorrent = new (await import('webtorrent')).default({
+    natUpnp: false,
+    natPmp: false,
+    utp: false
+  } as any)
+
+  return new Promise<string>((res, rej) => {
+    let file: TorrentFile
+
+    const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
+
+    const options = { path: directoryPath }
+    const torrent = webtorrent.add(torrentId, options, torrent => {
+      if (torrent.files.length !== 1) {
+        if (timer) clearTimeout(timer)
+
+        for (const file of torrent.files) {
+          deleteDownloadedFile({ directoryPath, filepath: file.path })
+        }
+
+        return safeWebtorrentDestroy(webtorrent, torrentId, undefined, target.torrentName)
+          .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))
+      }
+
+      logger.debug('Got torrent from webtorrent %s.', id, { infoHash: torrent.infoHash })
+
+      file = torrent.files[0]
+
+      // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed
+      const writeStream = createWriteStream(path)
+      writeStream.on('finish', () => {
+        if (timer) clearTimeout(timer)
+
+        safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName)
+          .then(() => res(path))
+          .catch(err => logger.error('Cannot destroy webtorrent.', { err }))
+      })
+
+      pipeline(
+        file.createReadStream(),
+        writeStream,
+        err => {
+          if (err) rej(err)
+        }
+      )
+    })
+
+    torrent.on('error', err => rej(err))
+
+    timer = setTimeout(() => {
+      const err = new Error('Webtorrent download timeout.')
+
+      safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
+        .then(() => rej(err))
+        .catch(destroyErr => {
+          logger.error('Cannot destroy webtorrent.', { err: destroyErr })
+          rej(err)
+        })
+
+    }, timeout)
+  })
+}
+
+function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
+  return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => {
+    return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath)
+  })
+}
+
+async function createTorrentAndSetInfoHashFromPath (
+  videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
+  videoFile: MVideoFile,
+  filePath: string
+) {
+  const video = extractVideo(videoOrPlaylist)
+
+  const options = {
+    // Keep the extname, it's used by the client to stream the file inside a web browser
+    name: buildInfoName(video, videoFile),
+    createdBy: 'PeerTube',
+    announceList: buildAnnounceList(),
+    urlList: buildUrlList(video, videoFile)
+  }
+
+  const torrentContent = await createTorrentPromise(filePath, options)
+
+  const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
+  const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
+  logger.info('Creating torrent %s.', torrentPath)
+
+  await writeFile(torrentPath, torrentContent)
+
+  // Remove old torrent file if it existed
+  if (videoFile.hasTorrent()) {
+    await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
+  }
+
+  // FIXME: typings: parseTorrent now returns an async result
+  const parsedTorrent = await (parseTorrent(torrentContent) as unknown as Promise<parseTorrent.Instance>)
+  videoFile.infoHash = parsedTorrent.infoHash
+  videoFile.torrentFilename = torrentFilename
+}
+
+async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
+  const video = extractVideo(videoOrPlaylist)
+
+  if (!videoFile.torrentFilename) {
+    logger.error(`Video file ${videoFile.filename} of video ${video.uuid} doesn't have a torrent file, skipping torrent metadata update`)
+    return
+  }
+
+  const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
+
+  if (!await pathExists(oldTorrentPath)) {
+    logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath)
+    return
+  }
+
+  const torrentContent = await readFile(oldTorrentPath)
+  const decoded = bencode.decode(torrentContent)
+
+  decoded['announce-list'] = buildAnnounceList()
+  decoded.announce = decoded['announce-list'][0][0]
+
+  decoded['url-list'] = buildUrlList(video, videoFile)
+
+  decoded.info.name = buildInfoName(video, videoFile)
+  decoded['creation date'] = Math.ceil(Date.now() / 1000)
+
+  const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
+  const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename)
+
+  logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath)
+
+  await writeFile(newTorrentPath, bencode.encode(decoded))
+  await remove(oldTorrentPath)
+
+  videoFile.torrentFilename = newTorrentFilename
+  videoFile.infoHash = sha1(bencode.encode(decoded.info))
+}
+
+function generateMagnetUri (
+  video: MVideo,
+  videoFile: MVideoFileRedundanciesOpt,
+  trackerUrls: string[]
+) {
+  const xs = videoFile.getTorrentUrl()
+  const announce = trackerUrls
+
+  let urlList = video.hasPrivateStaticPath()
+    ? []
+    : [ videoFile.getFileUrl(video) ]
+
+  const redundancies = videoFile.RedundancyVideos
+  if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
+
+  const magnetHash = {
+    xs,
+    announce,
+    urlList,
+    infoHash: videoFile.infoHash,
+    name: video.name
+  }
+
+  return magnetUriEncode(magnetHash)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  createTorrentPromise,
+  updateTorrentMetadata,
+
+  createTorrentAndSetInfoHash,
+  createTorrentAndSetInfoHashFromPath,
+
+  generateMagnetUri,
+  downloadWebTorrentVideo
+}
+
+// ---------------------------------------------------------------------------
+
+function safeWebtorrentDestroy (
+  webtorrent: Instance,
+  torrentId: string,
+  downloadedFile?: { directoryPath: string, filepath: string },
+  torrentName?: string
+) {
+  return new Promise<void>(res => {
+    webtorrent.destroy(err => {
+      // Delete torrent file
+      if (torrentName) {
+        logger.debug('Removing %s torrent after webtorrent download.', torrentId)
+        remove(torrentId)
+          .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
+      }
+
+      // Delete downloaded file
+      if (downloadedFile) deleteDownloadedFile(downloadedFile)
+
+      if (err) logger.warn('Cannot destroy webtorrent in timeout.', { err })
+
+      return res()
+    })
+  })
+}
+
+function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath: string }) {
+  // We want to delete the base directory
+  let pathToDelete = dirname(downloadedFile.filepath)
+  if (pathToDelete === '.') pathToDelete = downloadedFile.filepath
+
+  const toRemovePath = join(downloadedFile.directoryPath, pathToDelete)
+
+  logger.debug('Removing %s after webtorrent download.', toRemovePath)
+  remove(toRemovePath)
+    .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err }))
+}
+
+function buildAnnounceList () {
+  return [
+    [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
+    [ WEBSERVER.URL + '/tracker/announce' ]
+  ]
+}
+
+function buildUrlList (video: MVideo, videoFile: MVideoFile) {
+  if (video.hasPrivateStaticPath()) return []
+
+  return [ videoFile.getFileUrl(video) ]
+}
+
+function buildInfoName (video: MVideo, videoFile: MVideoFile) {
+  const videoName = video.name.replace(/[/\\?%*:|"<>]/g, '-')
+
+  return `${videoName} ${videoFile.resolution}p${videoFile.extname}`
+}

server/core/helpers/youtube-dl/index.ts

@@ -0,0 +1,3 @@
+export * from './youtube-dl-cli.js'
+export * from './youtube-dl-info-builder.js'
+export * from './youtube-dl-wrapper.js'

server/core/helpers/youtube-dl/youtube-dl-cli.ts

@@ -0,0 +1,262 @@
+import { execa, Options as ExecaNodeOptions } from 'execa'
+import { ensureDir, pathExists } from 'fs-extra/esm'
+import { writeFile } from 'fs/promises'
+import { OptionsOfBufferResponseBody } from 'got'
+import { dirname, join } from 'path'
+import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { logger, loggerTagsFactory } from '../logger.js'
+import { getProxy, isProxyEnabled } from '../proxy.js'
+import { isBinaryResponse, peertubeGot } from '../requests.js'
+
+type ProcessOptions = Pick<ExecaNodeOptions, 'cwd' | 'maxBuffer'>
+
+const lTags = loggerTagsFactory('youtube-dl')
+
+const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
+
+export class YoutubeDLCLI {
+
+  static async safeGet () {
+    if (!await pathExists(youtubeDLBinaryPath)) {
+      await ensureDir(dirname(youtubeDLBinaryPath))
+
+      await this.updateYoutubeDLBinary()
+    }
+
+    return new YoutubeDLCLI()
+  }
+
+  static async updateYoutubeDLBinary () {
+    const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
+
+    logger.info('Updating youtubeDL binary from %s.', url, lTags())
+
+    const gotOptions: OptionsOfBufferResponseBody = {
+      context: { bodyKBLimit: 20_000 },
+      responseType: 'buffer' as 'buffer'
+    }
+
+    if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {
+      gotOptions.headers = {
+        authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN
+      }
+    }
+
+    try {
+      let gotResult = await peertubeGot(url, gotOptions)
+
+      if (!isBinaryResponse(gotResult)) {
+        const json = JSON.parse(gotResult.body.toString())
+        const latest = json.filter(release => release.prerelease === false)[0]
+        if (!latest) throw new Error('Cannot find latest release')
+
+        const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
+        const releaseAsset = latest.assets.find(a => a.name === releaseName)
+        if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
+
+        gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
+      }
+
+      if (!isBinaryResponse(gotResult)) {
+        throw new Error('Not a binary response')
+      }
+
+      await writeFile(youtubeDLBinaryPath, gotResult.body)
+
+      logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
+    } catch (err) {
+      logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
+    }
+  }
+
+  static getYoutubeDLVideoFormat (enabledResolutions: VideoResolutionType[], useBestFormat: boolean) {
+    /**
+     * list of format selectors in order or preference
+     * see https://github.com/ytdl-org/youtube-dl#format-selection
+     *
+     * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
+     * of being able to do a "quick-transcode"
+     * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
+     * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
+     *
+     * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
+     **/
+
+    let result: string[] = []
+
+    if (!useBestFormat) {
+      const resolution = enabledResolutions.length === 0
+        ? VideoResolution.H_720P
+        : Math.max(...enabledResolutions)
+
+      result = [
+        `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
+        `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
+        `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
+      ]
+    }
+
+    return result.concat([
+      'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
+      'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
+      'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
+      'best' // Ultimate fallback
+    ]).join('/')
+  }
+
+  private constructor () {
+
+  }
+
+  download (options: {
+    url: string
+    format: string
+    output: string
+    processOptions: ProcessOptions
+    timeout?: number
+    additionalYoutubeDLArgs?: string[]
+  }) {
+    let args = options.additionalYoutubeDLArgs || []
+    args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
+
+    return this.run({
+      url: options.url,
+      processOptions: options.processOptions,
+      timeout: options.timeout,
+      args
+    })
+  }
+
+  async getInfo (options: {
+    url: string
+    format: string
+    processOptions: ProcessOptions
+    additionalYoutubeDLArgs?: string[]
+  }) {
+    const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
+
+    const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
+
+    const data = await this.run({ url, args: completeArgs, processOptions })
+    if (!data) return undefined
+
+    const info = data.map(d => JSON.parse(d))
+
+    return info.length === 1
+      ? info[0]
+      : info
+  }
+
+  async getListInfo (options: {
+    url: string
+    latestVideosCount?: number
+    processOptions: ProcessOptions
+  }): Promise<{ upload_date: string, webpage_url: string }[]> {
+    const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
+
+    if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
+      // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
+      additionalYoutubeDLArgs.push('--flat-playlist')
+    }
+
+    if (options.latestVideosCount !== undefined) {
+      additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
+    }
+
+    const result = await this.getInfo({
+      url: options.url,
+      format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
+      processOptions: options.processOptions,
+      additionalYoutubeDLArgs
+    })
+
+    if (!result) return result
+    if (!Array.isArray(result)) return [ result ]
+
+    return result
+  }
+
+  async getSubs (options: {
+    url: string
+    format: 'vtt'
+    processOptions: ProcessOptions
+  }) {
+    const { url, format, processOptions } = options
+
+    const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
+
+    const data = await this.run({ url, args, processOptions })
+    const files: string[] = []
+
+    const skipString = '[info] Writing video subtitles to: '
+
+    for (let i = 0, len = data.length; i < len; i++) {
+      const line = data[i]
+
+      if (line.indexOf(skipString) === 0) {
+        files.push(line.slice(skipString.length))
+      }
+    }
+
+    return files
+  }
+
+  private async run (options: {
+    url: string
+    args: string[]
+    timeout?: number
+    processOptions: ProcessOptions
+  }) {
+    const { url, args, timeout, processOptions } = options
+
+    let completeArgs = this.wrapWithProxyOptions(args)
+    completeArgs = this.wrapWithIPOptions(completeArgs)
+    completeArgs = this.wrapWithFFmpegOptions(completeArgs)
+
+    const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
+    const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
+
+    if (timeout) {
+      setTimeout(() => subProcess.kill(), timeout)
+    }
+
+    const output = await subProcess
+
+    logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
+
+    return output.stdout
+      ? output.stdout.trim().split(/\r?\n/)
+      : undefined
+  }
+
+  private wrapWithProxyOptions (args: string[]) {
+    if (isProxyEnabled()) {
+      logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
+
+      return [ '--proxy', getProxy() ].concat(args)
+    }
+
+    return args
+  }
+
+  private wrapWithIPOptions (args: string[]) {
+    if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
+      logger.debug('Force ipv4 for YoutubeDL')
+
+      return [ '--force-ipv4' ].concat(args)
+    }
+
+    return args
+  }
+
+  private wrapWithFFmpegOptions (args: string[]) {
+    if (process.env.FFMPEG_PATH) {
+      logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
+
+      return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
+    }
+
+    return args
+  }
+}

server/core/helpers/youtube-dl/youtube-dl-info-builder.ts

@@ -0,0 +1,207 @@
+import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants.js'
+import { peertubeTruncate } from '../core-utils.js'
+import { isUrlValid } from '../custom-validators/activitypub/misc.js'
+import { isArray } from '../custom-validators/misc.js'
+
+export type YoutubeDLInfo = {
+  name?: string
+  description?: string
+  category?: number
+  language?: string
+  licence?: number
+  nsfw?: boolean
+  tags?: string[]
+  thumbnailUrl?: string
+  ext?: string
+  originallyPublishedAtWithoutTime?: Date
+  webpageUrl?: string
+
+  urls?: string[]
+
+  chapters?: {
+    timecode: number
+    title: string
+  }[]
+}
+
+export class YoutubeDLInfoBuilder {
+  private readonly info: any
+
+  constructor (info: any) {
+    this.info = { ...info }
+  }
+
+  getInfo () {
+    const obj = this.buildVideoInfo(this.normalizeObject(this.info))
+    if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
+
+    return obj
+  }
+
+  private normalizeObject (obj: any) {
+    const newObj: any = {}
+
+    for (const key of Object.keys(obj)) {
+      // Deprecated key
+      if (key === 'resolution') continue
+
+      const value = obj[key]
+
+      if (typeof value === 'string') {
+        newObj[key] = value.normalize()
+      } else {
+        newObj[key] = value
+      }
+    }
+
+    return newObj
+  }
+
+  private buildOriginallyPublishedAt (obj: any) {
+    let originallyPublishedAt: Date = null
+
+    const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
+    if (uploadDateMatcher) {
+      originallyPublishedAt = new Date()
+      originallyPublishedAt.setHours(0, 0, 0, 0)
+
+      const year = parseInt(uploadDateMatcher[1], 10)
+      // Month starts from 0
+      const month = parseInt(uploadDateMatcher[2], 10) - 1
+      const day = parseInt(uploadDateMatcher[3], 10)
+
+      originallyPublishedAt.setFullYear(year, month, day)
+    }
+
+    return originallyPublishedAt
+  }
+
+  private buildVideoInfo (obj: any): YoutubeDLInfo {
+    return {
+      name: this.titleTruncation(obj.title),
+      description: this.descriptionTruncation(obj.description),
+      category: this.getCategory(obj.categories),
+      licence: this.getLicence(obj.license),
+      language: this.getLanguage(obj.language),
+      nsfw: this.isNSFW(obj),
+      tags: this.getTags(obj.tags),
+      thumbnailUrl: obj.thumbnail || undefined,
+      urls: this.buildAvailableUrl(obj),
+      originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
+      ext: obj.ext,
+      webpageUrl: obj.webpage_url,
+      chapters: isArray(obj.chapters)
+        ? obj.chapters.map((c: { start_time: number, title: string }) => ({ timecode: c.start_time, title: c.title }))
+        : []
+    }
+  }
+
+  private buildAvailableUrl (obj: any) {
+    const urls: string[] = []
+
+    if (obj.url) urls.push(obj.url)
+    if (obj.urls) {
+      if (Array.isArray(obj.urls)) urls.push(...obj.urls)
+      else urls.push(obj.urls)
+    }
+
+    const formats = Array.isArray(obj.formats)
+      ? obj.formats
+      : []
+
+    for (const format of formats) {
+      if (!format.url) continue
+
+      urls.push(format.url)
+    }
+
+    const thumbnails = Array.isArray(obj.thumbnails)
+      ? obj.thumbnails
+      : []
+
+    for (const thumbnail of thumbnails) {
+      if (!thumbnail.url) continue
+
+      urls.push(thumbnail.url)
+    }
+
+    if (obj.thumbnail) urls.push(obj.thumbnail)
+
+    for (const subtitleKey of Object.keys(obj.subtitles || {})) {
+      const subtitles = obj.subtitles[subtitleKey]
+      if (!Array.isArray(subtitles)) continue
+
+      for (const subtitle of subtitles) {
+        if (!subtitle.url) continue
+
+        urls.push(subtitle.url)
+      }
+    }
+
+    return urls.filter(u => u && isUrlValid(u))
+  }
+
+  private titleTruncation (title: string) {
+    return peertubeTruncate(title, {
+      length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
+      separator: /,? +/,
+      omission: ' […]'
+    })
+  }
+
+  private descriptionTruncation (description: string) {
+    if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
+
+    return peertubeTruncate(description, {
+      length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
+      separator: /,? +/,
+      omission: ' […]'
+    })
+  }
+
+  private isNSFW (info: any) {
+    return info?.age_limit >= 16
+  }
+
+  private getTags (tags: string[]) {
+    if (Array.isArray(tags) === false) return []
+
+    return tags
+      .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
+      .map(t => t.normalize())
+      .slice(0, 5)
+  }
+
+  private getLicence (licence: string) {
+    if (!licence) return undefined
+
+    if (licence.includes('Creative Commons Attribution')) return 1
+
+    for (const key of Object.keys(VIDEO_LICENCES)) {
+      const peertubeLicence = VIDEO_LICENCES[key]
+      if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
+    }
+
+    return undefined
+  }
+
+  private getCategory (categories: string[]) {
+    if (!categories) return undefined
+
+    const categoryString = categories[0]
+    if (!categoryString || typeof categoryString !== 'string') return undefined
+
+    if (categoryString === 'News & Politics') return 11
+
+    for (const key of Object.keys(VIDEO_CATEGORIES)) {
+      const category = VIDEO_CATEGORIES[key]
+      if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
+    }
+
+    return undefined
+  }
+
+  private getLanguage (language: string) {
+    return VIDEO_LANGUAGES[language] ? language : undefined
+  }
+}

server/core/helpers/youtube-dl/youtube-dl-wrapper.ts

@@ -0,0 +1,156 @@
+import { move, pathExists, remove } from 'fs-extra/esm'
+import { readdir } from 'fs/promises'
+import { dirname, join } from 'path'
+import { inspect } from 'util'
+import { VideoResolutionType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { isVideoFileExtnameValid } from '../custom-validators/videos.js'
+import { logger, loggerTagsFactory } from '../logger.js'
+import { generateVideoImportTmpPath } from '../utils.js'
+import { YoutubeDLCLI } from './youtube-dl-cli.js'
+import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder.js'
+
+const lTags = loggerTagsFactory('youtube-dl')
+
+export type YoutubeDLSubs = {
+  language: string
+  filename: string
+  path: string
+}[]
+
+const processOptions = {
+  maxBuffer: 1024 * 1024 * 30 // 30MB
+}
+
+class YoutubeDLWrapper {
+
+  constructor (
+    private readonly url: string,
+    private readonly enabledResolutions: VideoResolutionType[],
+    private readonly useBestFormat: boolean
+  ) {
+
+  }
+
+  async getInfoForDownload (youtubeDLArgs: string[] = []): Promise<YoutubeDLInfo> {
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const info = await youtubeDL.getInfo({
+      url: this.url,
+      format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
+      additionalYoutubeDLArgs: youtubeDLArgs,
+      processOptions
+    })
+
+    if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`)
+
+    if (info.is_live === true) throw new Error('Cannot download a live streaming.')
+
+    const infoBuilder = new YoutubeDLInfoBuilder(info)
+
+    return infoBuilder.getInfo()
+  }
+
+  async getInfoForListImport (options: {
+    latestVideosCount?: number
+  }) {
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const list = await youtubeDL.getListInfo({
+      url: this.url,
+      latestVideosCount: options.latestVideosCount,
+      processOptions
+    })
+
+    if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
+
+    return list.map(info => info.webpage_url)
+  }
+
+  async getSubtitles (): Promise<YoutubeDLSubs> {
+    const cwd = CONFIG.STORAGE.TMP_DIR
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } })
+    if (!files) return []
+
+    logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() })
+
+    const subtitles = files.reduce((acc, filename) => {
+      const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
+      if (!matched?.[1]) return acc
+
+      return [
+        ...acc,
+        {
+          language: matched[1],
+          path: join(cwd, filename),
+          filename
+        }
+      ]
+    }, [])
+
+    return subtitles
+  }
+
+  async downloadVideo (fileExt: string, timeout: number): Promise<string> {
+    // Leave empty the extension, youtube-dl will add it
+    const pathWithoutExtension = generateVideoImportTmpPath(this.url, '')
+
+    logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags())
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    try {
+      await youtubeDL.download({
+        url: this.url,
+        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
+        output: pathWithoutExtension,
+        timeout,
+        processOptions
+      })
+
+      // If youtube-dl did not guess an extension for our file, just use .mp4 as default
+      if (await pathExists(pathWithoutExtension)) {
+        await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
+      }
+
+      return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+    } catch (err) {
+      this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+        .then(path => {
+          logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
+
+          return remove(path)
+        })
+        .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
+
+      throw err
+    }
+  }
+
+  private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
+    if (!isVideoFileExtnameValid(sourceExt)) {
+      throw new Error('Invalid video extension ' + sourceExt)
+    }
+
+    const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
+
+    for (const extension of extensions) {
+      const path = tmpPath + extension
+
+      if (await pathExists(path)) return path
+    }
+
+    const directoryContent = await readdir(dirname(tmpPath))
+
+    throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`)
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  YoutubeDLWrapper
+}