ニジカ投稿局 https://tv.nizika.tv
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

657 lines
17 KiB

  1. import { ActivityVideoUrlObject, FileStorage, VideoResolution, type FileStorageType } from '@peertube/peertube-models'
  2. import { logger } from '@server/helpers/logger.js'
  3. import { extractVideo } from '@server/helpers/video.js'
  4. import { CONFIG } from '@server/initializers/config.js'
  5. import { buildRemoteUrl } from '@server/lib/activitypub/url.js'
  6. import {
  7. getHLSPrivateFileUrl,
  8. getObjectStoragePublicFileUrl,
  9. getWebVideoPrivateFileUrl
  10. } from '@server/lib/object-storage/index.js'
  11. import { getFSTorrentFilePath } from '@server/lib/paths.js'
  12. import { getVideoFileMimeType } from '@server/lib/video-file.js'
  13. import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js'
  14. import { MStreamingPlaylistVideo, MVideo, MVideoWithHost, isStreamingPlaylist } from '@server/types/models/index.js'
  15. import { remove } from 'fs-extra/esm'
  16. import memoizee from 'memoizee'
  17. import { join } from 'path'
  18. import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
  19. import {
  20. AllowNull,
  21. BelongsTo,
  22. Column,
  23. CreatedAt,
  24. DataType,
  25. Default,
  26. DefaultScope,
  27. ForeignKey,
  28. HasMany,
  29. Is, Scopes,
  30. Table,
  31. UpdatedAt
  32. } from 'sequelize-typescript'
  33. import validator from 'validator'
  34. import {
  35. isVideoFPSResolutionValid,
  36. isVideoFileExtnameValid,
  37. isVideoFileInfoHashValid,
  38. isVideoFileResolutionValid,
  39. isVideoFileSizeValid
  40. } from '../../helpers/custom-validators/videos.js'
  41. import {
  42. LAZY_STATIC_PATHS,
  43. MEMOIZE_LENGTH,
  44. MEMOIZE_TTL,
  45. STATIC_DOWNLOAD_PATHS,
  46. STATIC_PATHS,
  47. WEBSERVER
  48. } from '../../initializers/constants.js'
  49. import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file.js'
  50. import { VideoRedundancyModel } from '../redundancy/video-redundancy.js'
  51. import { SequelizeModel, doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js'
  52. import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js'
  53. import { VideoModel } from './video.js'
  54. export enum ScopeNames {
  55. WITH_VIDEO = 'WITH_VIDEO',
  56. WITH_METADATA = 'WITH_METADATA',
  57. WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST'
  58. }
  59. @DefaultScope(() => ({
  60. attributes: {
  61. exclude: [ 'metadata' ]
  62. }
  63. }))
  64. @Scopes(() => ({
  65. [ScopeNames.WITH_VIDEO]: {
  66. include: [
  67. {
  68. model: VideoModel.unscoped(),
  69. required: true
  70. }
  71. ]
  72. },
  73. [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => {
  74. return {
  75. include: [
  76. {
  77. model: VideoModel.unscoped(),
  78. required: false,
  79. where: options.whereVideo
  80. },
  81. {
  82. model: VideoStreamingPlaylistModel.unscoped(),
  83. required: false,
  84. include: [
  85. {
  86. model: VideoModel.unscoped(),
  87. required: true,
  88. where: options.whereVideo
  89. }
  90. ]
  91. }
  92. ]
  93. }
  94. },
  95. [ScopeNames.WITH_METADATA]: {
  96. attributes: {
  97. include: [ 'metadata' ]
  98. }
  99. }
  100. }))
  101. @Table({
  102. tableName: 'videoFile',
  103. indexes: [
  104. {
  105. fields: [ 'videoId' ],
  106. where: {
  107. videoId: {
  108. [Op.ne]: null
  109. }
  110. }
  111. },
  112. {
  113. fields: [ 'videoStreamingPlaylistId' ],
  114. where: {
  115. videoStreamingPlaylistId: {
  116. [Op.ne]: null
  117. }
  118. }
  119. },
  120. {
  121. fields: [ 'infoHash' ]
  122. },
  123. {
  124. fields: [ 'torrentFilename' ],
  125. unique: true
  126. },
  127. {
  128. fields: [ 'filename' ],
  129. unique: true
  130. },
  131. {
  132. fields: [ 'videoId', 'resolution', 'fps' ],
  133. unique: true,
  134. where: {
  135. videoId: {
  136. [Op.ne]: null
  137. }
  138. }
  139. },
  140. {
  141. fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ],
  142. unique: true,
  143. where: {
  144. videoStreamingPlaylistId: {
  145. [Op.ne]: null
  146. }
  147. }
  148. }
  149. ]
  150. })
  151. export class VideoFileModel extends SequelizeModel<VideoFileModel> {
  152. @CreatedAt
  153. createdAt: Date
  154. @UpdatedAt
  155. updatedAt: Date
  156. @AllowNull(false)
  157. @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution'))
  158. @Column
  159. resolution: number
  160. @AllowNull(true)
  161. @Column
  162. width: number
  163. @AllowNull(true)
  164. @Column
  165. height: number
  166. @AllowNull(false)
  167. @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size'))
  168. @Column(DataType.BIGINT)
  169. size: number
  170. @AllowNull(false)
  171. @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname'))
  172. @Column
  173. extname: string
  174. @AllowNull(true)
  175. @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
  176. @Column
  177. infoHash: string
  178. @AllowNull(false)
  179. @Default(-1)
  180. @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
  181. @Column
  182. fps: number
  183. @AllowNull(true)
  184. @Column(DataType.JSONB)
  185. metadata: any
  186. @AllowNull(true)
  187. @Column
  188. metadataUrl: string
  189. // Could be null for remote files
  190. @AllowNull(true)
  191. @Column
  192. fileUrl: string
  193. // Could be null for live files
  194. @AllowNull(true)
  195. @Column
  196. filename: string
  197. // Could be null for remote files
  198. @AllowNull(true)
  199. @Column
  200. torrentUrl: string
  201. // Could be null for live files
  202. @AllowNull(true)
  203. @Column
  204. torrentFilename: string
  205. @ForeignKey(() => VideoModel)
  206. @Column
  207. videoId: number
  208. @AllowNull(false)
  209. @Default(FileStorage.FILE_SYSTEM)
  210. @Column
  211. storage: FileStorageType
  212. @BelongsTo(() => VideoModel, {
  213. foreignKey: {
  214. allowNull: true
  215. },
  216. onDelete: 'CASCADE'
  217. })
  218. Video: Awaited<VideoModel>
  219. @ForeignKey(() => VideoStreamingPlaylistModel)
  220. @Column
  221. videoStreamingPlaylistId: number
  222. @BelongsTo(() => VideoStreamingPlaylistModel, {
  223. foreignKey: {
  224. allowNull: true
  225. },
  226. onDelete: 'CASCADE'
  227. })
  228. VideoStreamingPlaylist: Awaited<VideoStreamingPlaylistModel>
  229. @HasMany(() => VideoRedundancyModel, {
  230. foreignKey: {
  231. allowNull: true
  232. },
  233. onDelete: 'CASCADE',
  234. hooks: true
  235. })
  236. RedundancyVideos: Awaited<VideoRedundancyModel>[]
  237. static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist.bind(VideoFileModel), {
  238. promise: true,
  239. max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
  240. maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
  241. })
  242. static doesInfohashExist (infoHash: string) {
  243. const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
  244. return doesExist({ sequelize: this.sequelize, query, bind: { infoHash } })
  245. }
  246. static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
  247. const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
  248. return !!videoFile
  249. }
  250. static async doesOwnedTorrentFileExist (filename: string) {
  251. const query = 'SELECT 1 FROM "videoFile" ' +
  252. 'LEFT JOIN "video" "webvideo" ON "webvideo"."id" = "videoFile"."videoId" AND "webvideo"."remote" IS FALSE ' +
  253. 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
  254. 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
  255. 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webvideo"."id" IS NOT NULL) LIMIT 1'
  256. return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
  257. }
  258. static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
  259. const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
  260. `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
  261. return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
  262. }
  263. static loadByFilename (filename: string) {
  264. const query = {
  265. where: {
  266. filename
  267. }
  268. }
  269. return VideoFileModel.findOne(query)
  270. }
  271. static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
  272. const query = {
  273. where: {
  274. filename
  275. }
  276. }
  277. return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
  278. }
  279. static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
  280. const query = {
  281. where: {
  282. torrentFilename: filename
  283. }
  284. }
  285. return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
  286. }
  287. static load (id: number): Promise<MVideoFile> {
  288. return VideoFileModel.findByPk(id)
  289. }
  290. static loadWithMetadata (id: number) {
  291. return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
  292. }
  293. static loadWithVideo (id: number) {
  294. return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
  295. }
  296. static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
  297. const whereVideo = validator.default.isUUID(videoIdOrUUID + '')
  298. ? { uuid: videoIdOrUUID }
  299. : { id: videoIdOrUUID }
  300. const options = {
  301. where: {
  302. id
  303. }
  304. }
  305. return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] })
  306. .findOne(options)
  307. .then(file => {
  308. // We used `required: false` so check we have at least a video or a streaming playlist
  309. if (!file.Video && !file.VideoStreamingPlaylist) return null
  310. return file
  311. })
  312. }
  313. static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
  314. const query = {
  315. include: [
  316. {
  317. model: VideoModel.unscoped(),
  318. required: true,
  319. include: [
  320. {
  321. model: VideoStreamingPlaylistModel.unscoped(),
  322. required: true,
  323. where: {
  324. id: streamingPlaylistId
  325. }
  326. }
  327. ]
  328. }
  329. ],
  330. transaction
  331. }
  332. return VideoFileModel.findAll(query)
  333. }
  334. static getStats () {
  335. const webVideoFilesQuery: FindOptions = {
  336. include: [
  337. {
  338. attributes: [],
  339. required: true,
  340. model: VideoModel.unscoped(),
  341. where: {
  342. remote: false
  343. }
  344. }
  345. ]
  346. }
  347. const hlsFilesQuery: FindOptions = {
  348. include: [
  349. {
  350. attributes: [],
  351. required: true,
  352. model: VideoStreamingPlaylistModel.unscoped(),
  353. include: [
  354. {
  355. attributes: [],
  356. model: VideoModel.unscoped(),
  357. required: true,
  358. where: {
  359. remote: false
  360. }
  361. }
  362. ]
  363. }
  364. ]
  365. }
  366. return Promise.all([
  367. VideoFileModel.aggregate('size', 'SUM', webVideoFilesQuery),
  368. VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery)
  369. ]).then(([ webVideoResult, hlsResult ]) => ({
  370. totalLocalVideoFilesSize: parseAggregateResult(webVideoResult) + parseAggregateResult(hlsResult)
  371. }))
  372. }
  373. // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes
  374. static async customUpsert (
  375. videoFile: MVideoFile,
  376. mode: 'streaming-playlist' | 'video',
  377. transaction: Transaction
  378. ) {
  379. const baseFind = {
  380. fps: videoFile.fps,
  381. resolution: videoFile.resolution,
  382. transaction
  383. }
  384. const element = mode === 'streaming-playlist'
  385. ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
  386. : await VideoFileModel.loadWebVideoFile({ ...baseFind, videoId: videoFile.videoId })
  387. if (!element) return videoFile.save({ transaction })
  388. for (const k of Object.keys(videoFile.toJSON())) {
  389. element.set(k, videoFile[k])
  390. }
  391. return element.save({ transaction })
  392. }
  393. static async loadWebVideoFile (options: {
  394. videoId: number
  395. fps: number
  396. resolution: number
  397. transaction?: Transaction
  398. }) {
  399. const where = {
  400. fps: options.fps,
  401. resolution: options.resolution,
  402. videoId: options.videoId
  403. }
  404. return VideoFileModel.findOne({ where, transaction: options.transaction })
  405. }
  406. static async loadHLSFile (options: {
  407. playlistId: number
  408. fps: number
  409. resolution: number
  410. transaction?: Transaction
  411. }) {
  412. const where = {
  413. fps: options.fps,
  414. resolution: options.resolution,
  415. videoStreamingPlaylistId: options.playlistId
  416. }
  417. return VideoFileModel.findOne({ where, transaction: options.transaction })
  418. }
  419. static removeHLSFilesOfStreamingPlaylistId (videoStreamingPlaylistId: number) {
  420. const options = {
  421. where: { videoStreamingPlaylistId }
  422. }
  423. return VideoFileModel.destroy(options)
  424. }
  425. hasTorrent () {
  426. return this.infoHash && this.torrentFilename
  427. }
  428. getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
  429. if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
  430. return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
  431. }
  432. getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo {
  433. return extractVideo(this.getVideoOrStreamingPlaylist())
  434. }
  435. isAudio () {
  436. return this.resolution === VideoResolution.H_NOVIDEO
  437. }
  438. isLive () {
  439. return this.size === -1
  440. }
  441. isHLS () {
  442. return !!this.videoStreamingPlaylistId
  443. }
  444. // ---------------------------------------------------------------------------
  445. getObjectStorageUrl (video: MVideo) {
  446. if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
  447. return this.getPrivateObjectStorageUrl(video)
  448. }
  449. return this.getPublicObjectStorageUrl()
  450. }
  451. private getPrivateObjectStorageUrl (video: MVideo) {
  452. if (this.isHLS()) {
  453. return getHLSPrivateFileUrl(video, this.filename)
  454. }
  455. return getWebVideoPrivateFileUrl(this.filename)
  456. }
  457. private getPublicObjectStorageUrl () {
  458. if (this.isHLS()) {
  459. return getObjectStoragePublicFileUrl(this.fileUrl, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
  460. }
  461. return getObjectStoragePublicFileUrl(this.fileUrl, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
  462. }
  463. // ---------------------------------------------------------------------------
  464. getFileUrl (video: MVideo) {
  465. if (video.isOwned()) {
  466. if (this.storage === FileStorage.OBJECT_STORAGE) {
  467. return this.getObjectStorageUrl(video)
  468. }
  469. return WEBSERVER.URL + this.getFileStaticPath(video)
  470. }
  471. return this.fileUrl
  472. }
  473. // ---------------------------------------------------------------------------
  474. getFileStaticPath (video: MVideo) {
  475. if (this.isHLS()) return this.getHLSFileStaticPath(video)
  476. return this.getWebVideoFileStaticPath(video)
  477. }
  478. private getWebVideoFileStaticPath (video: MVideo) {
  479. if (isVideoInPrivateDirectory(video.privacy)) {
  480. return join(STATIC_PATHS.PRIVATE_WEB_VIDEOS, this.filename)
  481. }
  482. return join(STATIC_PATHS.WEB_VIDEOS, this.filename)
  483. }
  484. private getHLSFileStaticPath (video: MVideo) {
  485. if (isVideoInPrivateDirectory(video.privacy)) {
  486. return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
  487. }
  488. return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
  489. }
  490. // ---------------------------------------------------------------------------
  491. getFileDownloadUrl (video: MVideoWithHost) {
  492. const path = this.isHLS()
  493. ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
  494. : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
  495. if (video.isOwned()) return WEBSERVER.URL + path
  496. // FIXME: don't guess remote URL
  497. return buildRemoteUrl(video, path)
  498. }
  499. getRemoteTorrentUrl (video: MVideo) {
  500. if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
  501. return this.torrentUrl
  502. }
  503. // We proxify torrent requests so use a local URL
  504. getTorrentUrl () {
  505. if (!this.torrentFilename) return null
  506. return WEBSERVER.URL + this.getTorrentStaticPath()
  507. }
  508. getTorrentStaticPath () {
  509. if (!this.torrentFilename) return null
  510. return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename)
  511. }
  512. getTorrentDownloadUrl () {
  513. if (!this.torrentFilename) return null
  514. return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename)
  515. }
  516. removeTorrent () {
  517. if (!this.torrentFilename) return null
  518. const torrentPath = getFSTorrentFilePath(this)
  519. return remove(torrentPath)
  520. .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
  521. }
  522. hasSameUniqueKeysThan (other: MVideoFile) {
  523. return this.fps === other.fps &&
  524. this.resolution === other.resolution &&
  525. (
  526. (this.videoId !== null && this.videoId === other.videoId) ||
  527. (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
  528. )
  529. }
  530. withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
  531. if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
  532. return Object.assign(this, { Video: videoOrPlaylist })
  533. }
  534. // ---------------------------------------------------------------------------
  535. toActivityPubObject (this: MVideoFile, video: MVideo): ActivityVideoUrlObject {
  536. const mimeType = getVideoFileMimeType(this.extname, false)
  537. return {
  538. type: 'Link',
  539. mediaType: mimeType as ActivityVideoUrlObject['mediaType'],
  540. href: this.getFileUrl(video),
  541. height: this.height || this.resolution,
  542. width: this.width,
  543. size: this.size,
  544. fps: this.fps
  545. }
  546. }
  547. }