このコミットが含まれているのは:
@@ -0,0 +1,209 @@
|
||||
require 'json'
|
||||
require 'jwt'
|
||||
require 'net/http'
|
||||
require 'openssl'
|
||||
require 'tempfile'
|
||||
require 'uri'
|
||||
|
||||
|
||||
module GoogleDrive
|
||||
class ApiClient
|
||||
DRIVE_ENDPOINT = 'https://www.googleapis.com/drive/v3'
|
||||
TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token'
|
||||
FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
|
||||
DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.readonly'
|
||||
|
||||
def initialize service_account_email: ENV['GOOGLE_DRIVE_SERVICE_ACCOUNT_EMAIL'],
|
||||
private_key: ENV['GOOGLE_DRIVE_PRIVATE_KEY'],
|
||||
private_key_path: ENV['GOOGLE_DRIVE_PRIVATE_KEY_PATH'],
|
||||
subject: ENV['GOOGLE_DRIVE_SUBJECT']
|
||||
@service_account_email = service_account_email.to_s
|
||||
@private_key = load_private_key(private_key, private_key_path)
|
||||
@subject = subject.to_s.presence
|
||||
@access_token = nil
|
||||
end
|
||||
|
||||
def list_material_files_under_folder folder_id
|
||||
files = []
|
||||
walk_folder(folder_id, nil) do |entry, relative_path|
|
||||
next if entry['mimeType'] == FOLDER_MIME_TYPE
|
||||
|
||||
files << build_file_entry(entry, relative_path)
|
||||
end
|
||||
files
|
||||
end
|
||||
|
||||
def fetch_material_file file_id
|
||||
metadata = get_file(file_id)
|
||||
build_file_entry(metadata, metadata['name'])
|
||||
end
|
||||
|
||||
def download_to_tempfile file_id, filename:
|
||||
tempfile = Tempfile.new(['material-sync', File.extname(filename.to_s)])
|
||||
request_binary("/files/#{ file_id }",
|
||||
{ alt: 'media', supportsAllDrives: true }) do |chunk|
|
||||
tempfile.write(chunk)
|
||||
end
|
||||
tempfile.rewind
|
||||
tempfile
|
||||
rescue StandardError
|
||||
tempfile&.close!
|
||||
raise
|
||||
end
|
||||
|
||||
def extract_file_id value
|
||||
raw = value.to_s.strip
|
||||
return nil if raw.blank?
|
||||
return raw unless raw.include?('/')
|
||||
|
||||
uri = URI.parse(raw)
|
||||
return uri.query.to_s[%r{(?:^|&)id=([^&]+)}, 1] if uri.query.present?
|
||||
return uri.path[%r{/folders/([^/]+)}, 1] if uri.path.include?('/folders/')
|
||||
|
||||
uri.path[%r{/d/([^/]+)}, 1]
|
||||
rescue URI::InvalidURIError
|
||||
nil
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def walk_folder folder_id, prefix, &block
|
||||
list_children(folder_id).each do |entry|
|
||||
relative_path = MaterialSyncExportPath.build(prefix:,
|
||||
relative_path: entry['name'])
|
||||
if entry['mimeType'] == FOLDER_MIME_TYPE
|
||||
walk_folder(entry['id'], relative_path, &block)
|
||||
next
|
||||
end
|
||||
|
||||
block.call(entry, relative_path)
|
||||
end
|
||||
end
|
||||
|
||||
def build_file_entry entry, relative_path
|
||||
{ id: entry['id'],
|
||||
name: entry['name'],
|
||||
mime_type: entry['mimeType'],
|
||||
relative_path: MaterialSyncExportPath.normalize_path(relative_path),
|
||||
web_view_link: entry['webViewLink'],
|
||||
web_content_link: entry['webContentLink'] }
|
||||
end
|
||||
|
||||
def list_children folder_id
|
||||
files = []
|
||||
page_token = nil
|
||||
|
||||
loop do
|
||||
response =
|
||||
request_json('/files', {
|
||||
q: "'#{ folder_id }' in parents and trashed = false",
|
||||
fields: 'nextPageToken,files(id,name,mimeType,webViewLink,webContentLink)',
|
||||
orderBy: 'folder,name',
|
||||
pageSize: 1000,
|
||||
supportsAllDrives: true,
|
||||
includeItemsFromAllDrives: true,
|
||||
pageToken: page_token }.compact)
|
||||
files.concat(response.fetch('files'))
|
||||
page_token = response['nextPageToken']
|
||||
break if page_token.blank?
|
||||
end
|
||||
|
||||
files
|
||||
end
|
||||
|
||||
def get_file file_id
|
||||
request_json("/files/#{ file_id }",
|
||||
fields: 'id,name,mimeType,webViewLink,webContentLink',
|
||||
supportsAllDrives: true)
|
||||
end
|
||||
|
||||
def request_json path, params = {}
|
||||
response = request(:get, path, params:)
|
||||
unless response.is_a?(Net::HTTPSuccess)
|
||||
raise "Google Drive API error: #{ response.code } #{ response.body }"
|
||||
end
|
||||
|
||||
JSON.parse(response.body)
|
||||
end
|
||||
|
||||
def request_binary path, params = {}
|
||||
response = request(:get, path, params:)
|
||||
unless response.is_a?(Net::HTTPSuccess)
|
||||
raise "Google Drive download error: #{ response.code } #{ response.body }"
|
||||
end
|
||||
|
||||
response.read_body do |chunk|
|
||||
yield chunk
|
||||
end
|
||||
end
|
||||
|
||||
def request method, path, params: nil
|
||||
uri = URI(DRIVE_ENDPOINT + path)
|
||||
uri.query = URI.encode_www_form(params) if params.present?
|
||||
|
||||
klass =
|
||||
case method
|
||||
when :get
|
||||
Net::HTTP::Get
|
||||
else
|
||||
raise ArgumentError, "Unsupported Google Drive request method: #{ method }"
|
||||
end
|
||||
request = klass.new(uri)
|
||||
request['Authorization'] = "Bearer #{ access_token }"
|
||||
|
||||
Net::HTTP.start(uri.host,
|
||||
uri.port,
|
||||
use_ssl: true,
|
||||
open_timeout: 10,
|
||||
read_timeout: 60) do |http|
|
||||
http.request(request)
|
||||
end
|
||||
end
|
||||
|
||||
def access_token
|
||||
@access_token ||= fetch_access_token
|
||||
end
|
||||
|
||||
def fetch_access_token
|
||||
raise 'GOOGLE_DRIVE_SERVICE_ACCOUNT_EMAIL is required' if @service_account_email.blank?
|
||||
raise 'Google Drive private key is required' if @private_key.blank?
|
||||
|
||||
payload = { iss: @service_account_email,
|
||||
scope: DRIVE_SCOPE,
|
||||
aud: TOKEN_ENDPOINT,
|
||||
exp: 1.hour.from_now.to_i,
|
||||
iat: Time.current.to_i }
|
||||
payload[:sub] = @subject if @subject.present?
|
||||
|
||||
assertion = JWT.encode(payload, @private_key, 'RS256')
|
||||
uri = URI(TOKEN_ENDPOINT)
|
||||
request = Net::HTTP::Post.new(uri)
|
||||
request.set_form_data(
|
||||
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
|
||||
assertion:)
|
||||
|
||||
response =
|
||||
Net::HTTP.start(uri.host,
|
||||
uri.port,
|
||||
use_ssl: true,
|
||||
open_timeout: 10,
|
||||
read_timeout: 30) do |http|
|
||||
http.request(request)
|
||||
end
|
||||
unless response.is_a?(Net::HTTPSuccess)
|
||||
raise "Google OAuth error: #{ response.code } #{ response.body }"
|
||||
end
|
||||
|
||||
JSON.parse(response.body).fetch('access_token')
|
||||
end
|
||||
|
||||
def load_private_key private_key, private_key_path
|
||||
raw = private_key.to_s
|
||||
raw = raw.gsub('\n', "\n") if raw.present?
|
||||
raw = File.read(private_key_path) if raw.blank? && private_key_path.present?
|
||||
return nil if raw.blank?
|
||||
|
||||
OpenSSL::PKey::RSA.new(raw)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,56 @@
|
||||
class MaterialSyncExportPath
|
||||
class << self
|
||||
def build prefix: nil, relative_path: nil, filename: nil, source_file_id: nil
|
||||
path = normalize_path(relative_path)
|
||||
path = fallback_filename(filename, source_file_id) if path.blank?
|
||||
|
||||
normalized_prefix = normalize_path(prefix)
|
||||
return path if normalized_prefix.blank?
|
||||
return normalized_prefix if path.blank?
|
||||
|
||||
[normalized_prefix, path].join('/')
|
||||
end
|
||||
|
||||
def normalize_path path
|
||||
raw = path.to_s.delete("\0").tr('\\', '/').strip
|
||||
raw = raw.sub(%r{\A[A-Za-z]:/+}, '')
|
||||
raw = raw.sub(%r{\A/+}, '')
|
||||
return nil if raw.blank?
|
||||
|
||||
segments = raw.split('/').filter_map do |segment|
|
||||
normalized = normalize_segment(segment)
|
||||
normalized.presence
|
||||
end
|
||||
return nil if segments.empty?
|
||||
|
||||
segments.join('/')
|
||||
end
|
||||
|
||||
def uniquify path, source_file_id
|
||||
normalized = normalize_path(path)
|
||||
return normalized if normalized.blank? || source_file_id.blank?
|
||||
|
||||
ext = File.extname(normalized)
|
||||
base = ext.present? ? normalized.delete_suffix(ext) : normalized
|
||||
"#{ base }--#{ source_file_id }#{ ext }"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def fallback_filename filename, source_file_id
|
||||
name = normalize_path(filename)
|
||||
return name if name.present?
|
||||
return nil if source_file_id.blank?
|
||||
|
||||
source_file_id.to_s
|
||||
end
|
||||
|
||||
def normalize_segment segment
|
||||
cleaned = segment.to_s.delete("\0").tr('\\/', '_').strip
|
||||
return nil if cleaned.blank? || cleaned == '.'
|
||||
return '__' if cleaned == '..'
|
||||
|
||||
cleaned
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,5 +1,7 @@
|
||||
require 'digest'
|
||||
|
||||
class MaterialSyncImporter
|
||||
Result = Struct.new(:material, :suppressed, :suppression, keyword_init: true)
|
||||
Result = Struct.new(:material, :action, :suppressed, :suppression, keyword_init: true)
|
||||
|
||||
def self.import! attributes
|
||||
new(attributes).import!
|
||||
@@ -14,17 +16,53 @@ class MaterialSyncImporter
|
||||
key = normalized_source_key(source)
|
||||
raise ArgumentError, 'normalized_source_key is required for material sync' if key.blank?
|
||||
|
||||
suppression = MaterialSyncSuppressionMatcher.match(**source)
|
||||
return Result.new(material: nil, suppressed: true, suppression:) if suppression
|
||||
suppression = suppression_for(source)
|
||||
if suppression
|
||||
return Result.new(material: nil,
|
||||
action: :suppressed,
|
||||
suppressed: true,
|
||||
suppression:)
|
||||
end
|
||||
|
||||
material =
|
||||
Material
|
||||
.unscoped
|
||||
.find_or_initialize_by(normalized_source_key: key)
|
||||
material.assign_attributes(material_attributes.merge(source))
|
||||
material.save!
|
||||
event_type =
|
||||
if material.new_record?
|
||||
:create
|
||||
elsif material.discarded?
|
||||
:restore
|
||||
else
|
||||
:update
|
||||
end
|
||||
uploaded_blob = uploaded_blob!
|
||||
|
||||
Result.new(material:, suppressed: false, suppression: nil)
|
||||
Material.transaction do
|
||||
if material.persisted?
|
||||
MaterialVersionRecorder.ensure_snapshot!(material,
|
||||
created_by_user: @attributes[:updated_by_user])
|
||||
end
|
||||
material.assign_attributes(material_attributes.merge(source))
|
||||
material.discarded_at = nil if material.respond_to?(:discarded_at=)
|
||||
material.file.attach(uploaded_blob) if uploaded_blob
|
||||
material.save!
|
||||
upsert_export_item!(material)
|
||||
MaterialVersionRecorder.record!(material:,
|
||||
event_type:,
|
||||
created_by_user: @attributes[:updated_by_user])
|
||||
end
|
||||
MaterialThumbnailGenerator.generate!(material)
|
||||
|
||||
Result.new(material:,
|
||||
action: event_type == :create ? :imported : :updated,
|
||||
suppressed: false,
|
||||
suppression: nil)
|
||||
rescue StandardError
|
||||
uploaded_blob&.purge_later
|
||||
raise
|
||||
ensure
|
||||
close_tempfile!
|
||||
end
|
||||
|
||||
private
|
||||
@@ -44,6 +82,69 @@ class MaterialSyncImporter
|
||||
end
|
||||
|
||||
def material_attributes
|
||||
@attributes.except(:source_kind, :source_uri, :source_path, :source_file_id)
|
||||
@attributes.except(:source_kind, :source_uri, :source_path, :source_file_id,
|
||||
:file_blob, :file_tempfile, :filename, :content_type,
|
||||
:file_sha256, :export_path, :profile)
|
||||
end
|
||||
|
||||
def suppression_for source
|
||||
if source[:source_kind] == 'google_drive_file' && source[:source_path].present?
|
||||
return MaterialSyncSuppressionMatcher.match_google_drive_candidate(
|
||||
drive_file_id: source[:source_file_id],
|
||||
relative_path: source[:source_path])
|
||||
end
|
||||
|
||||
MaterialSyncSuppressionMatcher.match(**source)
|
||||
end
|
||||
|
||||
def uploaded_blob!
|
||||
return @attributes[:file_blob] if @attributes[:file_blob]
|
||||
|
||||
tempfile = @attributes[:file_tempfile]
|
||||
return nil unless tempfile
|
||||
|
||||
tempfile.rewind
|
||||
blob = ActiveStorage::Blob.create_and_upload!(
|
||||
io: tempfile,
|
||||
filename: @attributes[:filename],
|
||||
content_type: @attributes[:content_type])
|
||||
file_sha256 = @attributes[:file_sha256] || Digest::SHA256.file(tempfile.path).hexdigest
|
||||
blob.metadata['sha256'] = file_sha256 if file_sha256.present?
|
||||
blob.save! if blob.changed?
|
||||
tempfile.rewind
|
||||
blob
|
||||
end
|
||||
|
||||
def upsert_export_item! material
|
||||
export_path = @attributes[:export_path].to_s.strip
|
||||
return if export_path.blank?
|
||||
|
||||
profile = @attributes[:profile].presence || 'legacy_drive'
|
||||
path = export_path
|
||||
if export_path_taken_by_other?(material, profile, path)
|
||||
path = MaterialSyncExportPath.uniquify(path, @attributes[:source_file_id])
|
||||
end
|
||||
|
||||
item = material.material_export_items.find_or_initialize_by(profile:)
|
||||
item.export_path = path
|
||||
item.enabled = true
|
||||
item.created_by_user ||= @attributes[:created_by_user]
|
||||
item.save!
|
||||
end
|
||||
|
||||
def export_path_taken_by_other? material, profile, export_path
|
||||
MaterialExportItem
|
||||
.where(profile:, export_path:)
|
||||
.where.not(material_id: material.id)
|
||||
.exists?
|
||||
end
|
||||
|
||||
def close_tempfile!
|
||||
tempfile = @attributes[:file_tempfile]
|
||||
return unless tempfile
|
||||
|
||||
tempfile.close! unless tempfile.closed?
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
require 'digest'
|
||||
|
||||
class MaterialSyncRunner
|
||||
Result = Struct.new(:imported, :updated, :suppressed, :failed, :errors, keyword_init: true)
|
||||
|
||||
def self.sync_enabled!
|
||||
results = MaterialSyncSource.enabled.order(:id).map { |source| new(source).sync! }
|
||||
|
||||
Result.new(imported: results.sum(&:imported),
|
||||
updated: results.sum(&:updated),
|
||||
suppressed: results.sum(&:suppressed),
|
||||
failed: results.sum(&:failed),
|
||||
errors: results.flat_map(&:errors))
|
||||
end
|
||||
|
||||
def initialize source
|
||||
@source = source
|
||||
end
|
||||
|
||||
def sync!
|
||||
result = Result.new(imported: 0, updated: 0, suppressed: 0, failed: 0, errors: [])
|
||||
|
||||
candidates.each do |candidate|
|
||||
sync_candidate!(candidate, result)
|
||||
end
|
||||
|
||||
@source.update!(last_synced_at: Time.current)
|
||||
log_result(result)
|
||||
result
|
||||
rescue NotImplementedError, StandardError => e
|
||||
result.failed += 1
|
||||
result.errors << { source_id: @source.id, error: e.message }
|
||||
log_result(result)
|
||||
result
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def candidates
|
||||
case @source.source_kind
|
||||
when 'uri'
|
||||
[uri_candidate]
|
||||
when 'google_drive_path'
|
||||
google_drive_path_candidates
|
||||
when 'google_drive_file'
|
||||
[google_drive_file_candidate]
|
||||
when 'legacy_drive_path'
|
||||
raise NotImplementedError, 'legacy_drive_path material sync is not implemented'
|
||||
else
|
||||
raise NotImplementedError, "Unsupported material sync source_kind: #{ @source.source_kind }"
|
||||
end
|
||||
end
|
||||
|
||||
def uri_candidate
|
||||
{ source_kind: @source.source_kind,
|
||||
source_uri: @source.source_uri,
|
||||
source_path: @source.source_path,
|
||||
source_file_id: @source.source_file_id,
|
||||
url: @source.source_uri,
|
||||
tag: nil,
|
||||
created_by_user: @source.created_by_user,
|
||||
updated_by_user: @source.updated_by_user || @source.created_by_user }
|
||||
end
|
||||
|
||||
def sync_candidate! candidate, result
|
||||
block = MaterialImportBlockMatcher.match_for_sha256(candidate[:file_sha256])
|
||||
if block
|
||||
result.suppressed += 1
|
||||
Rails.logger.info(
|
||||
material_sync_log(action: 'suppressed',
|
||||
reason: block.reason,
|
||||
normalized_source_key: candidate_normalized_source_key(candidate)))
|
||||
return
|
||||
end
|
||||
|
||||
suppression = suppression_for(candidate)
|
||||
|
||||
if suppression
|
||||
result.suppressed += 1
|
||||
Rails.logger.info(
|
||||
material_sync_log(action: 'suppressed',
|
||||
normalized_source_key: suppression.normalized_source_key))
|
||||
return
|
||||
end
|
||||
|
||||
import = MaterialSyncImporter.import!(candidate)
|
||||
result.public_send("#{ import.action }=", result.public_send(import.action) + 1)
|
||||
rescue StandardError => e
|
||||
result.failed += 1
|
||||
result.errors << { source_id: @source.id,
|
||||
normalized_source_key: candidate_normalized_source_key(candidate),
|
||||
error: e.message }
|
||||
Rails.logger.warn(
|
||||
material_sync_log(action: 'failed',
|
||||
error: e.message,
|
||||
normalized_source_key: candidate_normalized_source_key(candidate)))
|
||||
ensure
|
||||
close_candidate_tempfile(candidate)
|
||||
end
|
||||
|
||||
def google_drive_path_candidates
|
||||
folder_id = google_drive_folder_id
|
||||
Enumerator.new do |entries|
|
||||
drive_client.list_material_files_under_folder(folder_id).each do |entry|
|
||||
entries << build_google_drive_candidate(entry)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def google_drive_file_candidate
|
||||
build_google_drive_candidate(drive_client.fetch_material_file(google_drive_file_id),
|
||||
single_file: true)
|
||||
end
|
||||
|
||||
def build_google_drive_candidate entry, single_file: false
|
||||
relative_path =
|
||||
if single_file
|
||||
nil
|
||||
else
|
||||
entry[:relative_path]
|
||||
end
|
||||
export_path =
|
||||
MaterialSyncExportPath.build(prefix: @source.export_path_prefix,
|
||||
relative_path:,
|
||||
filename: entry[:name],
|
||||
source_file_id: entry[:id])
|
||||
tempfile =
|
||||
drive_client.download_to_tempfile(entry[:id], filename: entry[:name])
|
||||
file_sha256 = Digest::SHA256.file(tempfile.path).hexdigest
|
||||
tempfile.rewind
|
||||
|
||||
{ source_kind: 'google_drive_file',
|
||||
source_uri: entry[:web_view_link] || google_drive_file_url(entry[:id]),
|
||||
source_path: relative_path.presence || MaterialSyncExportPath.normalize_path(entry[:name]),
|
||||
source_file_id: entry[:id],
|
||||
filename: entry[:name],
|
||||
content_type: entry[:mime_type],
|
||||
file_tempfile: tempfile,
|
||||
file_sha256:,
|
||||
export_path:,
|
||||
profile: @source.profile,
|
||||
tag: nil,
|
||||
url: nil,
|
||||
created_by_user: @source.created_by_user,
|
||||
updated_by_user: @source.updated_by_user || @source.created_by_user }
|
||||
end
|
||||
|
||||
def candidate_normalized_source_key candidate
|
||||
MaterialSyncSuppression.normalize_source_key(source_kind: candidate[:source_kind],
|
||||
source_uri: candidate[:source_uri],
|
||||
source_path: candidate[:source_path],
|
||||
source_file_id: candidate[:source_file_id])
|
||||
end
|
||||
|
||||
def suppression_for candidate
|
||||
if candidate[:source_kind] == 'google_drive_file' && candidate[:source_path].present?
|
||||
return MaterialSyncSuppressionMatcher.match_google_drive_candidate(
|
||||
drive_file_id: candidate[:source_file_id],
|
||||
relative_path: candidate[:source_path])
|
||||
end
|
||||
|
||||
MaterialSyncSuppressionMatcher.match(source_kind: candidate[:source_kind],
|
||||
source_uri: candidate[:source_uri],
|
||||
source_path: candidate[:source_path],
|
||||
source_file_id: candidate[:source_file_id])
|
||||
end
|
||||
|
||||
def drive_client
|
||||
@drive_client ||= GoogleDrive::ApiClient.new
|
||||
end
|
||||
|
||||
def google_drive_folder_id
|
||||
google_drive_target_id.tap do |id|
|
||||
raise ArgumentError, 'google_drive_path source_file_id or source_uri is required' if id.blank?
|
||||
end
|
||||
end
|
||||
|
||||
def google_drive_file_id
|
||||
google_drive_target_id.tap do |id|
|
||||
raise ArgumentError, 'google_drive_file source_file_id or source_uri is required' if id.blank?
|
||||
end
|
||||
end
|
||||
|
||||
def google_drive_target_id
|
||||
@source.source_file_id.presence || drive_client.extract_file_id(@source.source_uri)
|
||||
end
|
||||
|
||||
def google_drive_file_url file_id
|
||||
"https://drive.google.com/file/d/#{ file_id }/view"
|
||||
end
|
||||
|
||||
def close_candidate_tempfile candidate
|
||||
tempfile = candidate[:file_tempfile]
|
||||
return unless tempfile
|
||||
|
||||
tempfile.close! unless tempfile.closed?
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
|
||||
def log_result result
|
||||
Rails.logger.info(material_sync_log(imported: result.imported,
|
||||
updated: result.updated,
|
||||
suppressed: result.suppressed,
|
||||
failed: result.failed))
|
||||
end
|
||||
|
||||
def material_sync_log fields
|
||||
{ material_sync_source_id: @source.id,
|
||||
material_sync_source_name: @source.name }.merge(fields).to_json
|
||||
end
|
||||
end
|
||||
@@ -16,7 +16,53 @@ class MaterialSyncSuppressionMatcher
|
||||
MaterialSyncSuppression.find_by(normalized_source_key: key)
|
||||
end
|
||||
|
||||
def self.match_google_drive_candidate drive_file_id:, relative_path:
|
||||
normalized_path = validate_relative_path!(relative_path)
|
||||
|
||||
exact = [
|
||||
normalize_key('google_drive_file', drive_file_id),
|
||||
normalize_key('google_drive_path', normalized_path),
|
||||
normalize_key('legacy_drive_path', normalized_path),
|
||||
].compact
|
||||
|
||||
suppression =
|
||||
MaterialSyncSuppression.find_by(normalized_source_key: exact)
|
||||
return suppression if suppression
|
||||
|
||||
MaterialSyncSuppression
|
||||
.where(source_kind: ['google_drive_path_prefix', 'legacy_drive_path_prefix'])
|
||||
.where('drive_path = :path OR :path LIKE CONCAT(drive_path, "/%")', path: normalized_path)
|
||||
.order(:id)
|
||||
.first
|
||||
end
|
||||
|
||||
def self.suppressed?(...)
|
||||
match(...).present?
|
||||
end
|
||||
|
||||
def self.validate_relative_path! relative_path
|
||||
raw = relative_path.to_s.delete("\0").tr('\\', '/').strip
|
||||
raise ArgumentError, 'relative_path is required' if raw.blank?
|
||||
raise ArgumentError, 'relative_path must be relative' if raw.start_with?('/')
|
||||
raise ArgumentError, 'relative_path must be relative' if raw.match?(/\A[A-Za-z]:\//)
|
||||
if raw.start_with?('My Drive/', 'マイドライブ/')
|
||||
raise ArgumentError, 'relative_path must be relative to source folder'
|
||||
end
|
||||
raise ArgumentError, 'relative_path must not contain //' if raw.include?('//')
|
||||
raise ArgumentError, 'relative_path must not end with /' if raw.end_with?('/')
|
||||
|
||||
parts = raw.split('/')
|
||||
if parts.any? { |part| part.in?(['.', '..']) }
|
||||
raise ArgumentError, 'relative_path must not contain dot segments'
|
||||
end
|
||||
|
||||
raw
|
||||
end
|
||||
|
||||
def self.normalize_key source_kind, value
|
||||
return nil if value.blank?
|
||||
|
||||
MaterialSyncSuppression.normalize_source_key(source_kind:, source_path: value,
|
||||
source_file_id: value)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -23,10 +23,7 @@ class MaterialSyncSuppressionRegistrar
|
||||
private
|
||||
|
||||
def discard_existing_materials! suppression
|
||||
Material.unscoped
|
||||
.kept
|
||||
.where(normalized_source_key: suppression.normalized_source_key)
|
||||
.find_each do |material|
|
||||
matching_materials(suppression).find_each do |material|
|
||||
MaterialVersionRecorder.ensure_snapshot!(material, created_by_user: @created_by_user)
|
||||
material.discard!
|
||||
MaterialVersionRecorder.record!(material:,
|
||||
@@ -34,4 +31,21 @@ class MaterialSyncSuppressionRegistrar
|
||||
created_by_user: @created_by_user)
|
||||
end
|
||||
end
|
||||
|
||||
def matching_materials suppression
|
||||
materials = Material.unscoped.kept
|
||||
|
||||
case suppression.source_kind
|
||||
when 'google_drive_path', 'legacy_drive_path'
|
||||
materials.where(source_path: suppression.drive_path)
|
||||
when 'google_drive_path_prefix', 'legacy_drive_path_prefix'
|
||||
path = suppression.drive_path.to_s
|
||||
materials.where(
|
||||
'source_path = :path OR source_path LIKE :prefix',
|
||||
path:,
|
||||
prefix: "#{ path }/%")
|
||||
else
|
||||
materials.where(normalized_source_key: suppression.normalized_source_key)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -11,25 +11,30 @@ class MaterialThumbnailGenerator
|
||||
class << self
|
||||
def generate! material
|
||||
new(material).generate!
|
||||
rescue ActiveStorage::FileNotFoundError, ArgumentError, MiniMagick::Error => e
|
||||
Rails.logger.warn("Material thumbnail generation skipped: #{ e.class }: #{ e.message }")
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def initialize material
|
||||
@material = material
|
||||
@ffmpeg_stderr = []
|
||||
end
|
||||
|
||||
def generate!
|
||||
@material.thumbnail.purge if @material.thumbnail.attached?
|
||||
return unless @material.file.attached?
|
||||
return unless image? || video?
|
||||
return log_result(:no_file) unless @material.file.attached?
|
||||
return log_result(:unsupported_content_type) unless image? || video?
|
||||
|
||||
@material.file.blob.open do |file|
|
||||
thumbnail = image? ? image_thumbnail(file.path) : video_thumbnail(file.path)
|
||||
attach_thumbnail(thumbnail) if thumbnail
|
||||
return log_result(:generation_failed) unless thumbnail
|
||||
|
||||
return attach_thumbnail(thumbnail)
|
||||
end
|
||||
rescue ActiveStorage::FileNotFoundError => e
|
||||
log_result(:file_not_found, error: e)
|
||||
rescue MiniMagick::Error => e
|
||||
log_result(:mini_magick_error, error: e)
|
||||
rescue ArgumentError, StandardError => e
|
||||
log_result(:generation_failed, error: e)
|
||||
end
|
||||
|
||||
private
|
||||
@@ -38,7 +43,11 @@ class MaterialThumbnailGenerator
|
||||
|
||||
def video? = content_type.start_with?('video/')
|
||||
|
||||
def content_type = @material.file.blob.content_type.to_s
|
||||
def content_type
|
||||
return nil unless @material.file.attached?
|
||||
|
||||
@material.file.blob.content_type.to_s
|
||||
end
|
||||
|
||||
def image_thumbnail path
|
||||
image = MiniMagick::Image.open(path)
|
||||
@@ -71,16 +80,47 @@ class MaterialThumbnailGenerator
|
||||
'-frames:v', '1',
|
||||
'-f', 'image2',
|
||||
output_path)
|
||||
Rails.logger.warn("ffmpeg thumbnail failed: #{ stderr }") unless status.success?
|
||||
@ffmpeg_stderr << stderr if stderr.present?
|
||||
status.success?
|
||||
rescue Errno::ENOENT => e
|
||||
Rails.logger.warn("ffmpeg unavailable for material thumbnail: #{ e.message }")
|
||||
@ffmpeg_stderr << "ffmpeg unavailable: #{ e.message }"
|
||||
false
|
||||
end
|
||||
|
||||
def attach_thumbnail image
|
||||
@material.thumbnail.attach(io: File.open(image.path),
|
||||
filename: 'material-thumbnail.jpg',
|
||||
content_type: 'image/jpeg')
|
||||
blob = nil
|
||||
File.open(image.path) do |io|
|
||||
blob = ActiveStorage::Blob.create_and_upload!(
|
||||
io:,
|
||||
filename: 'material-thumbnail.jpg',
|
||||
content_type: 'image/jpeg')
|
||||
end
|
||||
@material.thumbnail.attach(blob)
|
||||
log_result(:attached)
|
||||
rescue StandardError => e
|
||||
blob&.purge_later
|
||||
log_result(:attach_failed, error: e)
|
||||
end
|
||||
|
||||
def log_result result, error: nil
|
||||
log_payload = { material_id: @material.id,
|
||||
file_blob_id: file_blob_id,
|
||||
content_type:,
|
||||
result:,
|
||||
error_class: error&.class&.name,
|
||||
error_message: error&.message,
|
||||
ffmpeg_stderr: @ffmpeg_stderr.join("\n").presence }
|
||||
if [:attached, :no_file, :unsupported_content_type].include?(result)
|
||||
Rails.logger.info("Material thumbnail generation: #{ log_payload.to_json }")
|
||||
else
|
||||
Rails.logger.warn("Material thumbnail generation: #{ log_payload.to_json }")
|
||||
end
|
||||
result
|
||||
end
|
||||
|
||||
def file_blob_id
|
||||
return nil unless @material.file.attached?
|
||||
|
||||
@material.file.blob.id
|
||||
end
|
||||
end
|
||||
|
||||
新しい課題から参照
ユーザをブロックする