ニジカ投稿局 https://tv.nizika.tv
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

youtube-dl-cli.ts 8.0 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. import { execa, Options as ExecaNodeOptions } from 'execa'
  2. import { ensureDir, pathExists } from 'fs-extra/esm'
  3. import { writeFile } from 'fs/promises'
  4. import { OptionsOfBufferResponseBody } from 'got'
  5. import { dirname, join } from 'path'
  6. import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models'
  7. import { CONFIG } from '@server/initializers/config.js'
  8. import { logger, loggerTagsFactory } from '../logger.js'
  9. import { getProxy, isProxyEnabled } from '../proxy.js'
  10. import { isBinaryResponse, peertubeGot } from '../requests.js'
  11. type ProcessOptions = Pick<ExecaNodeOptions, 'cwd' | 'maxBuffer'>
  12. const lTags = loggerTagsFactory('youtube-dl')
  13. const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
  14. export class YoutubeDLCLI {
  15. static async safeGet () {
  16. if (!await pathExists(youtubeDLBinaryPath)) {
  17. await ensureDir(dirname(youtubeDLBinaryPath))
  18. await this.updateYoutubeDLBinary()
  19. }
  20. return new YoutubeDLCLI()
  21. }
  22. static async updateYoutubeDLBinary () {
  23. const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
  24. logger.info('Updating youtubeDL binary from %s.', url, lTags())
  25. const gotOptions: OptionsOfBufferResponseBody = {
  26. context: { bodyKBLimit: 20_000 },
  27. responseType: 'buffer' as 'buffer'
  28. }
  29. if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {
  30. gotOptions.headers = {
  31. authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN
  32. }
  33. }
  34. try {
  35. let gotResult = await peertubeGot(url, gotOptions)
  36. if (!isBinaryResponse(gotResult)) {
  37. const json = JSON.parse(gotResult.body.toString())
  38. const latest = json.filter(release => release.prerelease === false)[0]
  39. if (!latest) throw new Error('Cannot find latest release')
  40. const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
  41. const releaseAsset = latest.assets.find(a => a.name === releaseName)
  42. if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
  43. gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
  44. }
  45. if (!isBinaryResponse(gotResult)) {
  46. throw new Error('Not a binary response')
  47. }
  48. await writeFile(youtubeDLBinaryPath, gotResult.body)
  49. logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
  50. } catch (err) {
  51. logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
  52. }
  53. }
  54. static getYoutubeDLVideoFormat (enabledResolutions: VideoResolutionType[], useBestFormat: boolean) {
  55. /**
  56. * list of format selectors in order or preference
  57. * see https://github.com/ytdl-org/youtube-dl#format-selection
  58. *
  59. * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
  60. * of being able to do a "quick-transcode"
  61. * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
  62. * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
  63. *
  64. * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
  65. **/
  66. let result: string[] = []
  67. if (!useBestFormat) {
  68. const resolution = enabledResolutions.length === 0
  69. ? VideoResolution.H_720P
  70. : Math.max(...enabledResolutions)
  71. result = [
  72. `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
  73. `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
  74. `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
  75. ]
  76. }
  77. return result.concat([
  78. 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
  79. 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
  80. 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
  81. 'best' // Ultimate fallback
  82. ]).join('/')
  83. }
  84. private constructor () {
  85. }
  86. download (options: {
  87. url: string
  88. format: string
  89. output: string
  90. processOptions: ProcessOptions
  91. timeout?: number
  92. additionalYoutubeDLArgs?: string[]
  93. }) {
  94. let args = options.additionalYoutubeDLArgs || []
  95. args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
  96. return this.run({
  97. url: options.url,
  98. processOptions: options.processOptions,
  99. timeout: options.timeout,
  100. args
  101. })
  102. }
  103. async getInfo (options: {
  104. url: string
  105. format: string
  106. processOptions: ProcessOptions
  107. additionalYoutubeDLArgs?: string[]
  108. }) {
  109. const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
  110. const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
  111. const data = await this.run({ url, args: completeArgs, processOptions })
  112. if (!data) return undefined
  113. const info = data.map(d => JSON.parse(d))
  114. return info.length === 1
  115. ? info[0]
  116. : info
  117. }
  118. async getListInfo (options: {
  119. url: string
  120. latestVideosCount?: number
  121. processOptions: ProcessOptions
  122. }): Promise<{ upload_date: string, webpage_url: string }[]> {
  123. const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
  124. if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
  125. // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
  126. additionalYoutubeDLArgs.push('--flat-playlist')
  127. }
  128. if (options.latestVideosCount !== undefined) {
  129. additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
  130. }
  131. const result = await this.getInfo({
  132. url: options.url,
  133. format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
  134. processOptions: options.processOptions,
  135. additionalYoutubeDLArgs
  136. })
  137. if (!result) return result
  138. if (!Array.isArray(result)) return [ result ]
  139. return result
  140. }
  141. async getSubs (options: {
  142. url: string
  143. format: 'vtt'
  144. processOptions: ProcessOptions
  145. }) {
  146. const { url, format, processOptions } = options
  147. const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
  148. const data = await this.run({ url, args, processOptions })
  149. const files: string[] = []
  150. const skipString = '[info] Writing video subtitles to: '
  151. for (let i = 0, len = data.length; i < len; i++) {
  152. const line = data[i]
  153. if (line.indexOf(skipString) === 0) {
  154. files.push(line.slice(skipString.length))
  155. }
  156. }
  157. return files
  158. }
  159. private async run (options: {
  160. url: string
  161. args: string[]
  162. timeout?: number
  163. processOptions: ProcessOptions
  164. }) {
  165. const { url, args, timeout, processOptions } = options
  166. let completeArgs = this.wrapWithProxyOptions(args)
  167. completeArgs = this.wrapWithIPOptions(completeArgs)
  168. completeArgs = this.wrapWithFFmpegOptions(completeArgs)
  169. const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
  170. const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
  171. if (timeout) {
  172. setTimeout(() => subProcess.kill(), timeout)
  173. }
  174. const output = await subProcess
  175. logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
  176. return output.stdout
  177. ? output.stdout.trim().split(/\r?\n/)
  178. : undefined
  179. }
  180. private wrapWithProxyOptions (args: string[]) {
  181. if (isProxyEnabled()) {
  182. logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
  183. return [ '--proxy', getProxy() ].concat(args)
  184. }
  185. return args
  186. }
  187. private wrapWithIPOptions (args: string[]) {
  188. if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
  189. logger.debug('Force ipv4 for YoutubeDL')
  190. return [ '--force-ipv4' ].concat(args)
  191. }
  192. return args
  193. }
  194. private wrapWithFFmpegOptions (args: string[]) {
  195. if (process.env.FFMPEG_PATH) {
  196. logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
  197. return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
  198. }
  199. return args
  200. }
  201. }