はじまりの大地

2024-07-15 09:14:04 +09:00
@@ -0,0 +1,5 @@
+export * from './keys.js'
+export * from './proxy.js'
+export * from './pre-signed-urls.js'
+export * from './urls.js'
+export * from './videos.js'
@@ -0,0 +1,22 @@
+import { join } from 'path'
+import { MStreamingPlaylistVideo } from '@server/types/models/index.js'
+
+export function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
+  return join(generateHLSObjectBaseStorageKey(playlist), filename)
+}
+
+export function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
+  return join(playlist.getStringType(), playlist.Video.uuid)
+}
+
+export function generateWebVideoObjectStorageKey (filename: string) {
+  return filename
+}
+
+export function generateOriginalVideoObjectStorageKey (filename: string) {
+  return filename
+}
+
+export function generateUserExportObjectStorageKey (filename: string) {
+  return filename
+}
@@ -0,0 +1,389 @@
+import { pipelinePromise } from '@server/helpers/core-utils.js'
+import { isArray } from '@server/helpers/custom-validators/misc.js'
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import Bluebird from 'bluebird'
+import { createReadStream, createWriteStream } from 'fs'
+import { ensureDir } from 'fs-extra/esm'
+import { dirname } from 'path'
+import { Readable } from 'stream'
+import { getInternalUrl } from './urls.js'
+import { getClient } from './shared/client.js'
+import { lTags } from './shared/logger.js'
+
+import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
+
+type BucketInfo = {
+  BUCKET_NAME: string
+  PREFIX?: string
+}
+
+async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo, continuationToken?: string) {
+  const s3Client = await getClient()
+
+  const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
+
+  const commandPrefix = bucketInfo.PREFIX + prefix
+  const listCommand = new ListObjectsV2Command({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Prefix: commandPrefix,
+    ContinuationToken: continuationToken
+  })
+
+  const listedObjects = await s3Client.send(listCommand)
+
+  if (isArray(listedObjects.Contents) !== true) return []
+
+  let keys = listedObjects.Contents.map(c => c.Key)
+
+  if (listedObjects.IsTruncated) {
+    keys = keys.concat(await listKeysOfPrefix(prefix, bucketInfo, listedObjects.NextContinuationToken))
+  }
+
+  return keys
+}
+
+// ---------------------------------------------------------------------------
+
+async function storeObject (options: {
+  inputPath: string
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}): Promise<string> {
+  const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options
+
+  logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
+
+  const fileStream = createReadStream(inputPath)
+
+  return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate })
+}
+
+async function storeContent (options: {
+  content: string
+  inputPath: string
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}): Promise<string> {
+  const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options
+
+  logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
+
+  return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate })
+}
+
+async function storeStream (options: {
+  stream: Readable
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}): Promise<string> {
+  const { stream, objectStorageKey, bucketInfo, isPrivate } = options
+
+  logger.debug('Streaming file to %s%s in bucket %s', bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
+
+  return uploadToStorage({ objectStorageKey, content: stream, bucketInfo, isPrivate })
+}
+
+// ---------------------------------------------------------------------------
+
+async function updateObjectACL (options: {
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { objectStorageKey, bucketInfo, isPrivate } = options
+
+  const acl = getACL(isPrivate)
+  if (!acl) return
+
+  const key = buildKey(objectStorageKey, bucketInfo)
+
+  logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
+
+  const { PutObjectAclCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new PutObjectAclCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: key,
+    ACL: acl
+  })
+
+  const client = await getClient()
+  await client.send(command)
+}
+
+async function updatePrefixACL (options: {
+  prefix: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { prefix, bucketInfo, isPrivate } = options
+
+  const acl = getACL(isPrivate)
+  if (!acl) return
+
+  const { PutObjectAclCommand } = await import('@aws-sdk/client-s3')
+
+  logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
+
+  return applyOnPrefix({
+    prefix,
+    bucketInfo,
+    commandBuilder: obj => {
+      logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
+
+      return new PutObjectAclCommand({
+        Bucket: bucketInfo.BUCKET_NAME,
+        Key: obj.Key,
+        ACL: acl
+      })
+    }
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
+  const key = buildKey(objectStorageKey, bucketInfo)
+
+  return removeObjectByFullKey(key, bucketInfo)
+}
+
+async function removeObjectByFullKey (fullKey: string, bucketInfo: Pick<BucketInfo, 'BUCKET_NAME'>) {
+  logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
+
+  const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new DeleteObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: fullKey
+  })
+
+  const client = await getClient()
+
+  return client.send(command)
+}
+
+async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
+  logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
+
+  const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
+
+  return applyOnPrefix({
+    prefix,
+    bucketInfo,
+    commandBuilder: obj => {
+      logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
+
+      return new DeleteObjectCommand({
+        Bucket: bucketInfo.BUCKET_NAME,
+        Key: obj.Key
+      })
+    }
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+async function makeAvailable (options: {
+  key: string
+  destination: string
+  bucketInfo: BucketInfo
+}) {
+  const { key, destination, bucketInfo } = options
+
+  await ensureDir(dirname(options.destination))
+
+  const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new GetObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(key, bucketInfo)
+  })
+
+  const client = await getClient()
+  const response = await client.send(command)
+
+  const file = createWriteStream(destination)
+  await pipelinePromise(response.Body as Readable, file)
+
+  file.close()
+}
+
+function buildKey (key: string, bucketInfo: BucketInfo) {
+  return bucketInfo.PREFIX + key
+}
+
+// ---------------------------------------------------------------------------
+
+async function createObjectReadStream (options: {
+  key: string
+  bucketInfo: BucketInfo
+  rangeHeader: string
+}) {
+  const { key, bucketInfo, rangeHeader } = options
+
+  const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new GetObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(key, bucketInfo),
+    Range: rangeHeader
+  })
+
+  const client = await getClient()
+  const response = await client.send(command)
+
+  return {
+    response,
+    stream: response.Body as Readable
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+async function getObjectStorageFileSize (options: {
+  key: string
+  bucketInfo: BucketInfo
+}) {
+  const { key, bucketInfo } = options
+
+  const { HeadObjectCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new HeadObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(key, bucketInfo)
+  })
+
+  const client = await getClient()
+  const response = await client.send(command)
+
+  return response.ContentLength
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  type BucketInfo,
+
+  buildKey,
+
+  storeObject,
+  storeContent,
+  storeStream,
+
+  removeObject,
+  removeObjectByFullKey,
+  removePrefix,
+
+  makeAvailable,
+
+  updateObjectACL,
+  updatePrefixACL,
+
+  listKeysOfPrefix,
+  createObjectReadStream,
+
+  getObjectStorageFileSize
+}
+
+// ---------------------------------------------------------------------------
+
+async function uploadToStorage (options: {
+  content: Readable | string
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { content, objectStorageKey, bucketInfo, isPrivate } = options
+
+  const input: PutObjectCommandInput = {
+    Body: content,
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(objectStorageKey, bucketInfo)
+  }
+
+  const acl = getACL(isPrivate)
+  if (acl) input.ACL = acl
+
+  const { Upload } = await import('@aws-sdk/lib-storage')
+
+  const parallelUploads3 = new Upload({
+    client: await getClient(),
+    queueSize: 4,
+    partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART,
+
+    // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts
+    // More detailed explanation:
+    // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274
+    // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928
+    leavePartsOnError: true,
+    params: input
+  })
+
+  const response = await parallelUploads3.done()
+  // Check is needed even if the HTTP status code is 200 OK
+  // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
+  if (!response.Bucket) {
+    const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}`
+    logger.error(message, { response, ...lTags() })
+    throw new Error(message)
+  }
+
+  logger.debug(
+    'Completed %s%s in bucket %s',
+    bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata }
+  )
+
+  return getInternalUrl(bucketInfo, objectStorageKey)
+}
+
+async function applyOnPrefix (options: {
+  prefix: string
+  bucketInfo: BucketInfo
+  commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0]
+
+  continuationToken?: string
+}) {
+  const { prefix, bucketInfo, commandBuilder, continuationToken } = options
+
+  const s3Client = await getClient()
+
+  const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
+
+  const commandPrefix = buildKey(prefix, bucketInfo)
+  const listCommand = new ListObjectsV2Command({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Prefix: commandPrefix,
+    ContinuationToken: continuationToken
+  })
+
+  const listedObjects = await s3Client.send(listCommand)
+
+  if (isArray(listedObjects.Contents) !== true) {
+    const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
+
+    logger.error(message, { response: listedObjects, ...lTags() })
+    throw new Error(message)
+  }
+
+  await Bluebird.map(listedObjects.Contents, object => {
+    const command = commandBuilder(object)
+
+    return s3Client.send(command)
+  }, { concurrency: 10 })
+
+  // Repeat if not all objects could be listed at once (limit of 1000?)
+  if (listedObjects.IsTruncated) {
+    await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken })
+  }
+}
+
+function getACL (isPrivate: boolean) {
+  return isPrivate
+    ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE as ObjectCannedACL
+    : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC as ObjectCannedACL
+}
@@ -0,0 +1,95 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { MStreamingPlaylistVideo, MUserExport, MVideoFile } from '@server/types/models/index.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
+import {
+  generateHLSObjectStorageKey,
+  generateOriginalVideoObjectStorageKey,
+  generateUserExportObjectStorageKey,
+  generateWebVideoObjectStorageKey
+} from './keys.js'
+import { buildKey, getClient } from './shared/index.js'
+import { getObjectStoragePublicFileUrl } from './urls.js'
+
+export async function generateWebVideoPresignedUrl (options: {
+  file: MVideoFile
+  downloadFilename: string
+}) {
+  const { file, downloadFilename } = options
+
+  const url = await generatePresignedUrl({
+    bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME,
+    key: buildKey(generateWebVideoObjectStorageKey(file.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS),
+    downloadFilename
+  })
+
+  return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
+}
+
+export async function generateHLSFilePresignedUrl (options: {
+  streamingPlaylist: MStreamingPlaylistVideo
+  file: MVideoFile
+  downloadFilename: string
+}) {
+  const { streamingPlaylist, file, downloadFilename } = options
+
+  const url = await generatePresignedUrl({
+    bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME,
+    key: buildKey(generateHLSObjectStorageKey(streamingPlaylist, file.filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS),
+    downloadFilename
+  })
+
+  return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+export async function generateUserExportPresignedUrl (options: {
+  userExport: MUserExport
+  downloadFilename: string
+}) {
+  const { userExport, downloadFilename } = options
+
+  const url = await generatePresignedUrl({
+    bucket: CONFIG.OBJECT_STORAGE.USER_EXPORTS.BUCKET_NAME,
+    key: buildKey(generateUserExportObjectStorageKey(userExport.filename), CONFIG.OBJECT_STORAGE.USER_EXPORTS),
+    downloadFilename
+  })
+
+  return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
+}
+
+export async function generateOriginalFilePresignedUrl (options: {
+  videoSource: MVideoSource
+  downloadFilename: string
+}) {
+  const { videoSource, downloadFilename } = options
+
+  const url = await generatePresignedUrl({
+    bucket: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME,
+    key: buildKey(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES),
+    downloadFilename
+  })
+
+  return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function generatePresignedUrl (options: {
+  bucket: string
+  key: string
+  downloadFilename: string
+}) {
+  const { bucket, downloadFilename, key } = options
+
+  const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+  const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner')
+
+  const command = new GetObjectCommand({
+    Bucket: bucket,
+    Key: key,
+    ResponseContentDisposition: `attachment; filename="${encodeURI(downloadFilename)}"`
+  })
+
+  return getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 })
+}
@@ -0,0 +1,100 @@
+import express from 'express'
+import { PassThrough, pipeline } from 'stream'
+import { HttpStatusCode } from '@peertube/peertube-models'
+import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist.js'
+import { logger } from '@server/helpers/logger.js'
+import { StreamReplacer } from '@server/helpers/stream-replacer.js'
+import { MStreamingPlaylist, MVideo } from '@server/types/models/index.js'
+import { injectQueryToPlaylistUrls } from '../hls.js'
+import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos.js'
+
+import type { GetObjectCommandOutput } from '@aws-sdk/client-s3'
+
+export async function proxifyWebVideoFile (options: {
+  req: express.Request
+  res: express.Response
+  filename: string
+}) {
+  const { req, res, filename } = options
+
+  logger.debug('Proxifying Web Video file %s from object storage.', filename)
+
+  try {
+    const { response: s3Response, stream } = await getWebVideoFileReadStream({
+      filename,
+      rangeHeader: req.header('range')
+    })
+
+    setS3Headers(res, s3Response)
+
+    return stream.pipe(res)
+  } catch (err) {
+    return handleObjectStorageFailure(res, err)
+  }
+}
+
+export async function proxifyHLS (options: {
+  req: express.Request
+  res: express.Response
+  playlist: MStreamingPlaylist
+  video: MVideo
+  filename: string
+  reinjectVideoFileToken: boolean
+}) {
+  const { req, res, playlist, video, filename, reinjectVideoFileToken } = options
+
+  logger.debug('Proxifying HLS file %s from object storage.', filename)
+
+  try {
+    const { response: s3Response, stream } = await getHLSFileReadStream({
+      playlist: playlist.withVideo(video),
+      filename,
+      rangeHeader: req.header('range')
+    })
+
+    setS3Headers(res, s3Response)
+
+    const streamReplacer = reinjectVideoFileToken
+      ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))))
+      : new PassThrough()
+
+    return pipeline(
+      stream,
+      streamReplacer,
+      res,
+      err => {
+        if (!err) return
+
+        handleObjectStorageFailure(res, err)
+      }
+    )
+  } catch (err) {
+    return handleObjectStorageFailure(res, err)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function handleObjectStorageFailure (res: express.Response, err: Error) {
+  if (err.name === 'NoSuchKey') {
+    logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
+    return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
+  }
+
+  logger.error('Object storage failure', { err })
+
+  return res.fail({
+    status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
+    message: err.message,
+    type: err.name
+  })
+}
+
+function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) {
+  if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) {
+    res.setHeader('Content-Range', s3Response.ContentRange)
+    res.status(HttpStatusCode.PARTIAL_CONTENT_206)
+  }
+}
@@ -0,0 +1,79 @@
+import type { S3Client } from '@aws-sdk/client-s3'
+import { logger } from '@server/helpers/logger.js'
+import { isProxyEnabled } from '@server/helpers/proxy.js'
+import { getAgent } from '@server/helpers/requests.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { lTags } from './logger.js'
+
+async function getProxyRequestHandler () {
+  if (!isProxyEnabled()) return null
+
+  const { agent } = getAgent()
+
+  const { NodeHttpHandler } = await import('@smithy/node-http-handler')
+
+  return new NodeHttpHandler({
+    httpAgent: agent.http,
+    httpsAgent: agent.https
+  })
+}
+
+let endpointParsed: URL
+function getEndpointParsed () {
+  if (endpointParsed) return endpointParsed
+
+  endpointParsed = new URL(getEndpoint())
+
+  return endpointParsed
+}
+
+let s3ClientPromise: Promise<S3Client>
+function getClient () {
+  if (s3ClientPromise) return s3ClientPromise
+
+  s3ClientPromise = (async () => {
+    const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE
+
+    const { S3Client } = await import('@aws-sdk/client-s3')
+
+    const s3Client = new S3Client({
+      endpoint: getEndpoint(),
+      region: OBJECT_STORAGE.REGION,
+      credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID
+        ? {
+          accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
+          secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
+        }
+        : undefined,
+      requestHandler: await getProxyRequestHandler(),
+      maxAttempts: CONFIG.OBJECT_STORAGE.MAX_REQUEST_ATTEMPTS
+    })
+
+    logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
+
+    return s3Client
+  })()
+
+  return s3ClientPromise
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getEndpointParsed,
+  getClient
+}
+
+// ---------------------------------------------------------------------------
+
+let endpoint: string
+function getEndpoint () {
+  if (endpoint) return endpoint
+
+  const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT
+  endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://')
+    ? CONFIG.OBJECT_STORAGE.ENDPOINT
+    : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT
+
+  return endpoint
+}
@@ -0,0 +1,3 @@
+export * from './client.js'
+export * from './logger.js'
+export * from '../object-storage-helpers.js'
@@ -0,0 +1,7 @@
+import { loggerTagsFactory } from '@server/helpers/logger.js'
+
+const lTags = loggerTagsFactory('object-storage')
+
+export {
+  lTags
+}
@@ -0,0 +1,40 @@
+import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js'
+import { MVideoUUID } from '@server/types/models/index.js'
+import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js'
+
+export function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
+  return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
+}
+
+// ---------------------------------------------------------------------------
+
+export function getObjectStoragePublicFileUrl (fileUrl: string, objectStorageConfig: { BASE_URL: string }) {
+  const baseUrl = objectStorageConfig.BASE_URL
+  if (!baseUrl) return fileUrl
+
+  return replaceByBaseUrl(fileUrl, baseUrl)
+}
+
+// ---------------------------------------------------------------------------
+
+export function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
+  return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
+}
+
+export function getWebVideoPrivateFileUrl (filename: string) {
+  return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename
+}
+// ---------------------------------------------------------------------------
+
+function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {
+  if (baseUrl) return baseUrl
+
+  return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/`
+}
+
+const regex = new RegExp('https?://[^/]+')
+function replaceByBaseUrl (fileUrl: string, baseUrl: string) {
+  if (!fileUrl) return fileUrl
+
+  return fileUrl.replace(regex, baseUrl)
+}
@@ -0,0 +1,25 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { MUserExport } from '@server/types/models/index.js'
+import { generateUserExportObjectStorageKey } from './keys.js'
+import { getObjectStorageFileSize, removeObject, storeStream } from './shared/index.js'
+import { Readable } from 'stream'
+
+export function storeUserExportFile (stream: Readable, userExport: MUserExport) {
+  return storeStream({
+    stream,
+    objectStorageKey: generateUserExportObjectStorageKey(userExport.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.USER_EXPORTS,
+    isPrivate: true
+  })
+}
+
+export function removeUserExportObjectStorage (userExport: MUserExport) {
+  return removeObject(generateUserExportObjectStorageKey(userExport.filename), CONFIG.OBJECT_STORAGE.USER_EXPORTS)
+}
+
+export function getUserExportFileObjectStorageSize (userExport: MUserExport) {
+  return getObjectStorageFileSize({
+    key: generateUserExportObjectStorageKey(userExport.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.USER_EXPORTS
+  })
+}
@@ -0,0 +1,222 @@
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models/index.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
+import { basename, join } from 'path'
+import { getHLSDirectory } from '../paths.js'
+import { VideoPathManager } from '../video-path-manager.js'
+import {
+  generateHLSObjectBaseStorageKey,
+  generateHLSObjectStorageKey,
+  generateOriginalVideoObjectStorageKey,
+  generateWebVideoObjectStorageKey
+} from './keys.js'
+import {
+  createObjectReadStream,
+  lTags,
+  listKeysOfPrefix,
+  makeAvailable,
+  removeObject,
+  removeObjectByFullKey,
+  removePrefix,
+  storeContent,
+  storeObject,
+  updateObjectACL,
+  updatePrefixACL
+} from './shared/index.js'
+
+export function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
+  return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+// ---------------------------------------------------------------------------
+
+export function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
+  return storeObject({
+    inputPath: join(getHLSDirectory(playlist.Video), filename),
+    objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+export function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
+  return storeObject({
+    inputPath: path,
+    objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) {
+  return storeContent({
+    content,
+    inputPath: path,
+    objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export function storeWebVideoFile (video: MVideo, file: MVideoFile) {
+  return storeObject({
+    inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
+    objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
+    isPrivate: video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export function storeOriginalVideoFile (inputPath: string, filename: string) {
+  return storeObject({
+    inputPath,
+    objectStorageKey: generateOriginalVideoObjectStorageKey(filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES,
+    isPrivate: true
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
+  await updateObjectACL({
+    objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
+    isPrivate: video.hasPrivateStaticPath()
+  })
+}
+
+export async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
+  await updatePrefixACL({
+    prefix: generateHLSObjectBaseStorageKey(playlist),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
+  return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+export function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
+  return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+export function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
+  return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+export function removeHLSFileObjectStorageByFullKey (key: string) {
+  return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+// ---------------------------------------------------------------------------
+
+export function removeWebVideoObjectStorage (videoFile: MVideoFile) {
+  return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
+}
+
+// ---------------------------------------------------------------------------
+
+export function removeOriginalFileObjectStorage (videoSource: MVideoSource) {
+  return removeObject(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
+}
+
+// ---------------------------------------------------------------------------
+
+export async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
+  const key = generateHLSObjectStorageKey(playlist, filename)
+
+  logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
+
+  await makeAvailable({
+    key,
+    destination,
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
+  })
+
+  return destination
+}
+
+export async function makeWebVideoFileAvailable (filename: string, destination: string) {
+  const key = generateWebVideoObjectStorageKey(filename)
+
+  logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags())
+
+  await makeAvailable({
+    key,
+    destination,
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS
+  })
+
+  return destination
+}
+
+export async function makeOriginalFileAvailable (keptOriginalFilename: string, destination: string) {
+  const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
+
+  logger.info('Fetching Original Video file %s from object storage to %s.', key, destination, lTags())
+
+  await makeAvailable({
+    key,
+    destination,
+    bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES
+  })
+
+  return destination
+}
+
+// ---------------------------------------------------------------------------
+
+export function getWebVideoFileReadStream (options: {
+  filename: string
+  rangeHeader: string
+}) {
+  const { filename, rangeHeader } = options
+
+  const key = generateWebVideoObjectStorageKey(filename)
+
+  return createObjectReadStream({
+    key,
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
+    rangeHeader
+  })
+}
+
+export function getHLSFileReadStream (options: {
+  playlist: MStreamingPlaylistVideo
+  filename: string
+  rangeHeader: string
+}) {
+  const { playlist, filename, rangeHeader } = options
+
+  const key = generateHLSObjectStorageKey(playlist, filename)
+
+  return createObjectReadStream({
+    key,
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    rangeHeader
+  })
+}
+
+export function getOriginalFileReadStream (options: {
+  keptOriginalFilename: string
+  rangeHeader: string
+}) {
+  const { keptOriginalFilename, rangeHeader } = options
+
+  const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
+
+  return createObjectReadStream({
+    key,
+    bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES,
+    rangeHeader
+  })
+}