ニジカ投稿局 https://tv.nizika.tv
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

332 lines
12 KiB

  1. import { uniqify } from '@peertube/peertube-core-utils'
  2. import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
  3. import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js'
  4. import { listKeysOfPrefix, removeObjectByFullKey } from '@server/lib/object-storage/object-storage-helpers.js'
  5. import { UserExportModel } from '@server/models/user/user-export.js'
  6. import { StoryboardModel } from '@server/models/video/storyboard.js'
  7. import { VideoCaptionModel } from '@server/models/video/video-caption.js'
  8. import { VideoFileModel } from '@server/models/video/video-file.js'
  9. import { VideoSourceModel } from '@server/models/video/video-source.js'
  10. import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
  11. import Bluebird from 'bluebird'
  12. import { remove } from 'fs-extra/esm'
  13. import { readdir, stat } from 'fs/promises'
  14. import { basename, dirname, join } from 'path'
  15. import { getUUIDFromFilename } from '../core/helpers/utils.js'
  16. import { CONFIG } from '../core/initializers/config.js'
  17. import { initDatabaseModels } from '../core/initializers/database.js'
  18. import { ActorImageModel } from '../core/models/actor/actor-image.js'
  19. import { VideoRedundancyModel } from '../core/models/redundancy/video-redundancy.js'
  20. import { ThumbnailModel } from '../core/models/video/thumbnail.js'
  21. import { VideoModel } from '../core/models/video/video.js'
  22. import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js'
  23. run()
  24. .then(() => process.exit(0))
  25. .catch(err => {
  26. console.error(err)
  27. process.exit(-1)
  28. })
  29. async function run () {
  30. await initDatabaseModels(true)
  31. displayPeerTubeMustBeStoppedWarning()
  32. await new FSPruner().prune()
  33. console.log('\n')
  34. await new ObjectStoragePruner().prune()
  35. }
  36. // ---------------------------------------------------------------------------
  37. // Object storage
  38. // ---------------------------------------------------------------------------
  39. class ObjectStoragePruner {
  40. private readonly keysToDelete: { bucket: string, key: string }[] = []
  41. async prune () {
  42. if (!CONFIG.OBJECT_STORAGE.ENABLED) return
  43. console.log('Pruning object storage.')
  44. await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.WEB_VIDEOS, this.doesWebVideoFileExistFactory())
  45. await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory())
  46. await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory())
  47. await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory())
  48. if (this.keysToDelete.length === 0) {
  49. console.log('No unknown object storage files to delete.')
  50. return
  51. }
  52. const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n')
  53. console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`)
  54. const res = await askPruneConfirmation()
  55. if (res !== true) {
  56. console.log('Exiting without deleting object storage files.')
  57. return
  58. }
  59. console.log('Deleting object storage files...\n')
  60. for (const { bucket, key } of this.keysToDelete) {
  61. await removeObjectByFullKey(key, { BUCKET_NAME: bucket })
  62. }
  63. console.log(`${this.keysToDelete.length} object storage files deleted.`)
  64. }
  65. private async findFilesToDelete (
  66. config: { BUCKET_NAME: string, PREFIX?: string },
  67. existFun: (file: string) => Promise<boolean> | boolean
  68. ) {
  69. try {
  70. const keys = await listKeysOfPrefix('', config)
  71. await Bluebird.map(keys, async key => {
  72. if (await existFun(key) !== true) {
  73. this.keysToDelete.push({ bucket: config.BUCKET_NAME, key })
  74. }
  75. }, { concurrency: 20 })
  76. } catch (err) {
  77. const prefixMessage = config.PREFIX
  78. ? ` and prefix ${config.PREFIX}`
  79. : ''
  80. console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage)
  81. }
  82. }
  83. private doesWebVideoFileExistFactory () {
  84. return (key: string) => {
  85. const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
  86. return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
  87. }
  88. }
  89. private doesStreamingPlaylistFileExistFactory () {
  90. return (key: string) => {
  91. const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)))
  92. return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE)
  93. }
  94. }
  95. private doesOriginalFileExistFactory () {
  96. return (key: string) => {
  97. const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
  98. return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
  99. }
  100. }
  101. private doesUserExportFileExistFactory () {
  102. return (key: string) => {
  103. const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
  104. return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
  105. }
  106. }
  107. private sanitizeKey (key: string, config: { PREFIX: string }) {
  108. return key.replace(new RegExp(`^${config.PREFIX}`), '')
  109. }
  110. }
  111. // ---------------------------------------------------------------------------
  112. // FS
  113. // ---------------------------------------------------------------------------
  114. class FSPruner {
  115. private pathsToDelete: string[] = []
  116. async prune () {
  117. const dirs = Object.values(CONFIG.STORAGE)
  118. if (uniqify(dirs).length !== dirs.length) {
  119. console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  120. process.exit(0)
  121. }
  122. console.log('Pruning filesystem storage.')
  123. console.log('Detecting files to remove, it can take a while...')
  124. await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory())
  125. await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory())
  126. await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory())
  127. await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory())
  128. await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory())
  129. await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory())
  130. await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory())
  131. await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW))
  132. await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE))
  133. await this.findFilesToDelete(CONFIG.STORAGE.CAPTIONS_DIR, this.doesCaptionExistFactory())
  134. await this.findFilesToDelete(CONFIG.STORAGE.STORYBOARDS_DIR, this.doesStoryboardExistFactory())
  135. await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory())
  136. await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory())
  137. const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  138. this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ]
  139. if (this.pathsToDelete.length === 0) {
  140. console.log('No unknown filesystem files to delete.')
  141. return
  142. }
  143. const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n')
  144. console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`)
  145. const res = await askPruneConfirmation()
  146. if (res !== true) {
  147. console.log('Exiting without deleting filesystem files.')
  148. return
  149. }
  150. console.log('Deleting filesystem files...\n')
  151. for (const path of this.pathsToDelete) {
  152. await remove(path)
  153. }
  154. console.log(`${this.pathsToDelete.length} filesystem files deleted.`)
  155. }
  156. private async findFilesToDelete (directory: string, existFun: (file: string) => Promise<boolean> | boolean) {
  157. const files = await readdir(directory)
  158. await Bluebird.map(files, async file => {
  159. const filePath = join(directory, file)
  160. if (await existFun(filePath) !== true) {
  161. this.pathsToDelete.push(filePath)
  162. }
  163. }, { concurrency: 20 })
  164. }
  165. private doesWebVideoFileExistFactory () {
  166. return (filePath: string) => {
  167. // Don't delete private directory
  168. if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
  169. return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
  170. }
  171. }
  172. private doesHLSPlaylistExistFactory () {
  173. return (hlsPath: string) => {
  174. // Don't delete private directory
  175. if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
  176. return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM)
  177. }
  178. }
  179. private doesOriginalVideoExistFactory () {
  180. return (filePath: string) => {
  181. return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
  182. }
  183. }
  184. private doesTorrentFileExistFactory () {
  185. return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
  186. }
  187. private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
  188. return async (filePath: string) => {
  189. const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
  190. if (!thumbnail) return false
  191. if (keepOnlyOwned) {
  192. const video = await VideoModel.load(thumbnail.videoId)
  193. if (video.isOwned() === false) return false
  194. }
  195. return true
  196. }
  197. }
  198. private doesActorImageExistFactory () {
  199. return async (filePath: string) => {
  200. const image = await ActorImageModel.loadByFilename(basename(filePath))
  201. return !!image
  202. }
  203. }
  204. private doesStoryboardExistFactory () {
  205. return async (filePath: string) => {
  206. const storyboard = await StoryboardModel.loadByFilename(basename(filePath))
  207. return !!storyboard
  208. }
  209. }
  210. private doesCaptionExistFactory () {
  211. return async (filePath: string) => {
  212. const caption = await VideoCaptionModel.loadWithVideoByFilename(basename(filePath))
  213. return !!caption
  214. }
  215. }
  216. private doesRedundancyExistFactory () {
  217. return async (filePath: string) => {
  218. const isPlaylist = (await stat(filePath)).isDirectory()
  219. if (isPlaylist) {
  220. // Don't delete HLS redundancy directory
  221. if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
  222. const uuid = getUUIDFromFilename(filePath)
  223. const video = await VideoModel.loadWithFiles(uuid)
  224. if (!video) return false
  225. const p = video.getHLSPlaylist()
  226. if (!p) return false
  227. const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
  228. return !!redundancy
  229. }
  230. const file = await VideoFileModel.loadByFilename(basename(filePath))
  231. if (!file) return false
  232. const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
  233. return !!redundancy
  234. }
  235. }
  236. private doesUserExportExistFactory () {
  237. return (filePath: string) => {
  238. const filename = basename(filePath)
  239. // Only detect non-existing user export
  240. if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true
  241. return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM)
  242. }
  243. }
  244. }
  245. async function askPruneConfirmation () {
  246. return askConfirmation(
  247. 'These unknown files can be deleted, but please check your backups first (bugs happen). ' +
  248. 'Can we delete these files?'
  249. )
  250. }