このコミットが含まれているのは:
2026-06-26 00:21:33 +09:00
コミット 8304909c8c
26個のファイルの変更1251行の追加33行の削除
+209
ファイルの表示
@@ -0,0 +1,209 @@
require 'json'
require 'jwt'
require 'net/http'
require 'openssl'
require 'tempfile'
require 'uri'
module GoogleDrive
class ApiClient
DRIVE_ENDPOINT = 'https://www.googleapis.com/drive/v3'
TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token'
FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.readonly'
def initialize service_account_email: ENV['GOOGLE_DRIVE_SERVICE_ACCOUNT_EMAIL'],
private_key: ENV['GOOGLE_DRIVE_PRIVATE_KEY'],
private_key_path: ENV['GOOGLE_DRIVE_PRIVATE_KEY_PATH'],
subject: ENV['GOOGLE_DRIVE_SUBJECT']
@service_account_email = service_account_email.to_s
@private_key = load_private_key(private_key, private_key_path)
@subject = subject.to_s.presence
@access_token = nil
end
def list_material_files_under_folder folder_id
files = []
walk_folder(folder_id, nil) do |entry, relative_path|
next if entry['mimeType'] == FOLDER_MIME_TYPE
files << build_file_entry(entry, relative_path)
end
files
end
def fetch_material_file file_id
metadata = get_file(file_id)
build_file_entry(metadata, metadata['name'])
end
def download_to_tempfile file_id, filename:
tempfile = Tempfile.new(['material-sync', File.extname(filename.to_s)])
request_binary("/files/#{ file_id }",
{ alt: 'media', supportsAllDrives: true }) do |chunk|
tempfile.write(chunk)
end
tempfile.rewind
tempfile
rescue StandardError
tempfile&.close!
raise
end
def extract_file_id value
raw = value.to_s.strip
return nil if raw.blank?
return raw unless raw.include?('/')
uri = URI.parse(raw)
return uri.query.to_s[%r{(?:^|&)id=([^&]+)}, 1] if uri.query.present?
return uri.path[%r{/folders/([^/]+)}, 1] if uri.path.include?('/folders/')
uri.path[%r{/d/([^/]+)}, 1]
rescue URI::InvalidURIError
nil
end
private
def walk_folder folder_id, prefix, &block
list_children(folder_id).each do |entry|
relative_path = MaterialSyncExportPath.build(prefix:,
relative_path: entry['name'])
if entry['mimeType'] == FOLDER_MIME_TYPE
walk_folder(entry['id'], relative_path, &block)
next
end
block.call(entry, relative_path)
end
end
def build_file_entry entry, relative_path
{ id: entry['id'],
name: entry['name'],
mime_type: entry['mimeType'],
relative_path: MaterialSyncExportPath.normalize_path(relative_path),
web_view_link: entry['webViewLink'],
web_content_link: entry['webContentLink'] }
end
def list_children folder_id
files = []
page_token = nil
loop do
response =
request_json('/files', {
q: "'#{ folder_id }' in parents and trashed = false",
fields: 'nextPageToken,files(id,name,mimeType,webViewLink,webContentLink)',
orderBy: 'folder,name',
pageSize: 1000,
supportsAllDrives: true,
includeItemsFromAllDrives: true,
pageToken: page_token }.compact)
files.concat(response.fetch('files'))
page_token = response['nextPageToken']
break if page_token.blank?
end
files
end
def get_file file_id
request_json("/files/#{ file_id }",
fields: 'id,name,mimeType,webViewLink,webContentLink',
supportsAllDrives: true)
end
def request_json path, params = {}
response = request(:get, path, params:)
unless response.is_a?(Net::HTTPSuccess)
raise "Google Drive API error: #{ response.code } #{ response.body }"
end
JSON.parse(response.body)
end
def request_binary path, params = {}
response = request(:get, path, params:)
unless response.is_a?(Net::HTTPSuccess)
raise "Google Drive download error: #{ response.code } #{ response.body }"
end
response.read_body do |chunk|
yield chunk
end
end
def request method, path, params: nil
uri = URI(DRIVE_ENDPOINT + path)
uri.query = URI.encode_www_form(params) if params.present?
klass =
case method
when :get
Net::HTTP::Get
else
raise ArgumentError, "Unsupported Google Drive request method: #{ method }"
end
request = klass.new(uri)
request['Authorization'] = "Bearer #{ access_token }"
Net::HTTP.start(uri.host,
uri.port,
use_ssl: true,
open_timeout: 10,
read_timeout: 60) do |http|
http.request(request)
end
end
def access_token
@access_token ||= fetch_access_token
end
def fetch_access_token
raise 'GOOGLE_DRIVE_SERVICE_ACCOUNT_EMAIL is required' if @service_account_email.blank?
raise 'Google Drive private key is required' if @private_key.blank?
payload = { iss: @service_account_email,
scope: DRIVE_SCOPE,
aud: TOKEN_ENDPOINT,
exp: 1.hour.from_now.to_i,
iat: Time.current.to_i }
payload[:sub] = @subject if @subject.present?
assertion = JWT.encode(payload, @private_key, 'RS256')
uri = URI(TOKEN_ENDPOINT)
request = Net::HTTP::Post.new(uri)
request.set_form_data(
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
assertion:)
response =
Net::HTTP.start(uri.host,
uri.port,
use_ssl: true,
open_timeout: 10,
read_timeout: 30) do |http|
http.request(request)
end
unless response.is_a?(Net::HTTPSuccess)
raise "Google OAuth error: #{ response.code } #{ response.body }"
end
JSON.parse(response.body).fetch('access_token')
end
def load_private_key private_key, private_key_path
raw = private_key.to_s
raw = raw.gsub('\n', "\n") if raw.present?
raw = File.read(private_key_path) if raw.blank? && private_key_path.present?
return nil if raw.blank?
OpenSSL::PKey::RSA.new(raw)
end
end
end
+56
ファイルの表示
@@ -0,0 +1,56 @@
class MaterialSyncExportPath
class << self
def build prefix: nil, relative_path: nil, filename: nil, source_file_id: nil
path = normalize_path(relative_path)
path = fallback_filename(filename, source_file_id) if path.blank?
normalized_prefix = normalize_path(prefix)
return path if normalized_prefix.blank?
return normalized_prefix if path.blank?
[normalized_prefix, path].join('/')
end
def normalize_path path
raw = path.to_s.delete("\0").tr('\\', '/').strip
raw = raw.sub(%r{\A[A-Za-z]:/+}, '')
raw = raw.sub(%r{\A/+}, '')
return nil if raw.blank?
segments = raw.split('/').filter_map do |segment|
normalized = normalize_segment(segment)
normalized.presence
end
return nil if segments.empty?
segments.join('/')
end
def uniquify path, source_file_id
normalized = normalize_path(path)
return normalized if normalized.blank? || source_file_id.blank?
ext = File.extname(normalized)
base = ext.present? ? normalized.delete_suffix(ext) : normalized
"#{ base }--#{ source_file_id }#{ ext }"
end
private
def fallback_filename filename, source_file_id
name = normalize_path(filename)
return name if name.present?
return nil if source_file_id.blank?
source_file_id.to_s
end
def normalize_segment segment
cleaned = segment.to_s.delete("\0").tr('\\/', '_').strip
return nil if cleaned.blank? || cleaned == '.'
return '__' if cleaned == '..'
cleaned
end
end
end
+108 -7
ファイルの表示
@@ -1,5 +1,7 @@
require 'digest'
class MaterialSyncImporter
Result = Struct.new(:material, :suppressed, :suppression, keyword_init: true)
Result = Struct.new(:material, :action, :suppressed, :suppression, keyword_init: true)
def self.import! attributes
new(attributes).import!
@@ -14,17 +16,53 @@ class MaterialSyncImporter
key = normalized_source_key(source)
raise ArgumentError, 'normalized_source_key is required for material sync' if key.blank?
suppression = MaterialSyncSuppressionMatcher.match(**source)
return Result.new(material: nil, suppressed: true, suppression:) if suppression
suppression = suppression_for(source)
if suppression
return Result.new(material: nil,
action: :suppressed,
suppressed: true,
suppression:)
end
material =
Material
.unscoped
.find_or_initialize_by(normalized_source_key: key)
material.assign_attributes(material_attributes.merge(source))
material.save!
event_type =
if material.new_record?
:create
elsif material.discarded?
:restore
else
:update
end
uploaded_blob = uploaded_blob!
Result.new(material:, suppressed: false, suppression: nil)
Material.transaction do
if material.persisted?
MaterialVersionRecorder.ensure_snapshot!(material,
created_by_user: @attributes[:updated_by_user])
end
material.assign_attributes(material_attributes.merge(source))
material.discarded_at = nil if material.respond_to?(:discarded_at=)
material.file.attach(uploaded_blob) if uploaded_blob
material.save!
upsert_export_item!(material)
MaterialVersionRecorder.record!(material:,
event_type:,
created_by_user: @attributes[:updated_by_user])
end
MaterialThumbnailGenerator.generate!(material)
Result.new(material:,
action: event_type == :create ? :imported : :updated,
suppressed: false,
suppression: nil)
rescue StandardError
uploaded_blob&.purge_later
raise
ensure
close_tempfile!
end
private
@@ -44,6 +82,69 @@ class MaterialSyncImporter
end
def material_attributes
@attributes.except(:source_kind, :source_uri, :source_path, :source_file_id)
@attributes.except(:source_kind, :source_uri, :source_path, :source_file_id,
:file_blob, :file_tempfile, :filename, :content_type,
:file_sha256, :export_path, :profile)
end
def suppression_for source
if source[:source_kind] == 'google_drive_file' && source[:source_path].present?
return MaterialSyncSuppressionMatcher.match_google_drive_candidate(
drive_file_id: source[:source_file_id],
relative_path: source[:source_path])
end
MaterialSyncSuppressionMatcher.match(**source)
end
def uploaded_blob!
return @attributes[:file_blob] if @attributes[:file_blob]
tempfile = @attributes[:file_tempfile]
return nil unless tempfile
tempfile.rewind
blob = ActiveStorage::Blob.create_and_upload!(
io: tempfile,
filename: @attributes[:filename],
content_type: @attributes[:content_type])
file_sha256 = @attributes[:file_sha256] || Digest::SHA256.file(tempfile.path).hexdigest
blob.metadata['sha256'] = file_sha256 if file_sha256.present?
blob.save! if blob.changed?
tempfile.rewind
blob
end
def upsert_export_item! material
export_path = @attributes[:export_path].to_s.strip
return if export_path.blank?
profile = @attributes[:profile].presence || 'legacy_drive'
path = export_path
if export_path_taken_by_other?(material, profile, path)
path = MaterialSyncExportPath.uniquify(path, @attributes[:source_file_id])
end
item = material.material_export_items.find_or_initialize_by(profile:)
item.export_path = path
item.enabled = true
item.created_by_user ||= @attributes[:created_by_user]
item.save!
end
def export_path_taken_by_other? material, profile, export_path
MaterialExportItem
.where(profile:, export_path:)
.where.not(material_id: material.id)
.exists?
end
def close_tempfile!
tempfile = @attributes[:file_tempfile]
return unless tempfile
tempfile.close! unless tempfile.closed?
rescue StandardError
nil
end
end
+212
ファイルの表示
@@ -0,0 +1,212 @@
require 'digest'
class MaterialSyncRunner
Result = Struct.new(:imported, :updated, :suppressed, :failed, :errors, keyword_init: true)
def self.sync_enabled!
results = MaterialSyncSource.enabled.order(:id).map { |source| new(source).sync! }
Result.new(imported: results.sum(&:imported),
updated: results.sum(&:updated),
suppressed: results.sum(&:suppressed),
failed: results.sum(&:failed),
errors: results.flat_map(&:errors))
end
def initialize source
@source = source
end
def sync!
result = Result.new(imported: 0, updated: 0, suppressed: 0, failed: 0, errors: [])
candidates.each do |candidate|
sync_candidate!(candidate, result)
end
@source.update!(last_synced_at: Time.current)
log_result(result)
result
rescue NotImplementedError, StandardError => e
result.failed += 1
result.errors << { source_id: @source.id, error: e.message }
log_result(result)
result
end
private
def candidates
case @source.source_kind
when 'uri'
[uri_candidate]
when 'google_drive_path'
google_drive_path_candidates
when 'google_drive_file'
[google_drive_file_candidate]
when 'legacy_drive_path'
raise NotImplementedError, 'legacy_drive_path material sync is not implemented'
else
raise NotImplementedError, "Unsupported material sync source_kind: #{ @source.source_kind }"
end
end
def uri_candidate
{ source_kind: @source.source_kind,
source_uri: @source.source_uri,
source_path: @source.source_path,
source_file_id: @source.source_file_id,
url: @source.source_uri,
tag: nil,
created_by_user: @source.created_by_user,
updated_by_user: @source.updated_by_user || @source.created_by_user }
end
def sync_candidate! candidate, result
block = MaterialImportBlockMatcher.match_for_sha256(candidate[:file_sha256])
if block
result.suppressed += 1
Rails.logger.info(
material_sync_log(action: 'suppressed',
reason: block.reason,
normalized_source_key: candidate_normalized_source_key(candidate)))
return
end
suppression = suppression_for(candidate)
if suppression
result.suppressed += 1
Rails.logger.info(
material_sync_log(action: 'suppressed',
normalized_source_key: suppression.normalized_source_key))
return
end
import = MaterialSyncImporter.import!(candidate)
result.public_send("#{ import.action }=", result.public_send(import.action) + 1)
rescue StandardError => e
result.failed += 1
result.errors << { source_id: @source.id,
normalized_source_key: candidate_normalized_source_key(candidate),
error: e.message }
Rails.logger.warn(
material_sync_log(action: 'failed',
error: e.message,
normalized_source_key: candidate_normalized_source_key(candidate)))
ensure
close_candidate_tempfile(candidate)
end
def google_drive_path_candidates
folder_id = google_drive_folder_id
Enumerator.new do |entries|
drive_client.list_material_files_under_folder(folder_id).each do |entry|
entries << build_google_drive_candidate(entry)
end
end
end
def google_drive_file_candidate
build_google_drive_candidate(drive_client.fetch_material_file(google_drive_file_id),
single_file: true)
end
def build_google_drive_candidate entry, single_file: false
relative_path =
if single_file
nil
else
entry[:relative_path]
end
export_path =
MaterialSyncExportPath.build(prefix: @source.export_path_prefix,
relative_path:,
filename: entry[:name],
source_file_id: entry[:id])
tempfile =
drive_client.download_to_tempfile(entry[:id], filename: entry[:name])
file_sha256 = Digest::SHA256.file(tempfile.path).hexdigest
tempfile.rewind
{ source_kind: 'google_drive_file',
source_uri: entry[:web_view_link] || google_drive_file_url(entry[:id]),
source_path: relative_path.presence || MaterialSyncExportPath.normalize_path(entry[:name]),
source_file_id: entry[:id],
filename: entry[:name],
content_type: entry[:mime_type],
file_tempfile: tempfile,
file_sha256:,
export_path:,
profile: @source.profile,
tag: nil,
url: nil,
created_by_user: @source.created_by_user,
updated_by_user: @source.updated_by_user || @source.created_by_user }
end
def candidate_normalized_source_key candidate
MaterialSyncSuppression.normalize_source_key(source_kind: candidate[:source_kind],
source_uri: candidate[:source_uri],
source_path: candidate[:source_path],
source_file_id: candidate[:source_file_id])
end
def suppression_for candidate
if candidate[:source_kind] == 'google_drive_file' && candidate[:source_path].present?
return MaterialSyncSuppressionMatcher.match_google_drive_candidate(
drive_file_id: candidate[:source_file_id],
relative_path: candidate[:source_path])
end
MaterialSyncSuppressionMatcher.match(source_kind: candidate[:source_kind],
source_uri: candidate[:source_uri],
source_path: candidate[:source_path],
source_file_id: candidate[:source_file_id])
end
def drive_client
@drive_client ||= GoogleDrive::ApiClient.new
end
def google_drive_folder_id
google_drive_target_id.tap do |id|
raise ArgumentError, 'google_drive_path source_file_id or source_uri is required' if id.blank?
end
end
def google_drive_file_id
google_drive_target_id.tap do |id|
raise ArgumentError, 'google_drive_file source_file_id or source_uri is required' if id.blank?
end
end
def google_drive_target_id
@source.source_file_id.presence || drive_client.extract_file_id(@source.source_uri)
end
def google_drive_file_url file_id
"https://drive.google.com/file/d/#{ file_id }/view"
end
def close_candidate_tempfile candidate
tempfile = candidate[:file_tempfile]
return unless tempfile
tempfile.close! unless tempfile.closed?
rescue StandardError
nil
end
def log_result result
Rails.logger.info(material_sync_log(imported: result.imported,
updated: result.updated,
suppressed: result.suppressed,
failed: result.failed))
end
def material_sync_log fields
{ material_sync_source_id: @source.id,
material_sync_source_name: @source.name }.merge(fields).to_json
end
end
+46
ファイルの表示
@@ -16,7 +16,53 @@ class MaterialSyncSuppressionMatcher
MaterialSyncSuppression.find_by(normalized_source_key: key)
end
def self.match_google_drive_candidate drive_file_id:, relative_path:
normalized_path = validate_relative_path!(relative_path)
exact = [
normalize_key('google_drive_file', drive_file_id),
normalize_key('google_drive_path', normalized_path),
normalize_key('legacy_drive_path', normalized_path),
].compact
suppression =
MaterialSyncSuppression.find_by(normalized_source_key: exact)
return suppression if suppression
MaterialSyncSuppression
.where(source_kind: ['google_drive_path_prefix', 'legacy_drive_path_prefix'])
.where('drive_path = :path OR :path LIKE CONCAT(drive_path, "/%")', path: normalized_path)
.order(:id)
.first
end
def self.suppressed?(...)
match(...).present?
end
def self.validate_relative_path! relative_path
raw = relative_path.to_s.delete("\0").tr('\\', '/').strip
raise ArgumentError, 'relative_path is required' if raw.blank?
raise ArgumentError, 'relative_path must be relative' if raw.start_with?('/')
raise ArgumentError, 'relative_path must be relative' if raw.match?(/\A[A-Za-z]:\//)
if raw.start_with?('My Drive/', 'マイドライブ/')
raise ArgumentError, 'relative_path must be relative to source folder'
end
raise ArgumentError, 'relative_path must not contain //' if raw.include?('//')
raise ArgumentError, 'relative_path must not end with /' if raw.end_with?('/')
parts = raw.split('/')
if parts.any? { |part| part.in?(['.', '..']) }
raise ArgumentError, 'relative_path must not contain dot segments'
end
raw
end
def self.normalize_key source_kind, value
return nil if value.blank?
MaterialSyncSuppression.normalize_source_key(source_kind:, source_path: value,
source_file_id: value)
end
end
+18 -4
ファイルの表示
@@ -23,10 +23,7 @@ class MaterialSyncSuppressionRegistrar
private
def discard_existing_materials! suppression
Material.unscoped
.kept
.where(normalized_source_key: suppression.normalized_source_key)
.find_each do |material|
matching_materials(suppression).find_each do |material|
MaterialVersionRecorder.ensure_snapshot!(material, created_by_user: @created_by_user)
material.discard!
MaterialVersionRecorder.record!(material:,
@@ -34,4 +31,21 @@ class MaterialSyncSuppressionRegistrar
created_by_user: @created_by_user)
end
end
def matching_materials suppression
materials = Material.unscoped.kept
case suppression.source_kind
when 'google_drive_path', 'legacy_drive_path'
materials.where(source_path: suppression.drive_path)
when 'google_drive_path_prefix', 'legacy_drive_path_prefix'
path = suppression.drive_path.to_s
materials.where(
'source_path = :path OR source_path LIKE :prefix',
path:,
prefix: "#{ path }/%")
else
materials.where(normalized_source_key: suppression.normalized_source_key)
end
end
end
+53 -13
ファイルの表示
@@ -11,25 +11,30 @@ class MaterialThumbnailGenerator
class << self
def generate! material
new(material).generate!
rescue ActiveStorage::FileNotFoundError, ArgumentError, MiniMagick::Error => e
Rails.logger.warn("Material thumbnail generation skipped: #{ e.class }: #{ e.message }")
nil
end
end
def initialize material
@material = material
@ffmpeg_stderr = []
end
def generate!
@material.thumbnail.purge if @material.thumbnail.attached?
return unless @material.file.attached?
return unless image? || video?
return log_result(:no_file) unless @material.file.attached?
return log_result(:unsupported_content_type) unless image? || video?
@material.file.blob.open do |file|
thumbnail = image? ? image_thumbnail(file.path) : video_thumbnail(file.path)
attach_thumbnail(thumbnail) if thumbnail
return log_result(:generation_failed) unless thumbnail
return attach_thumbnail(thumbnail)
end
rescue ActiveStorage::FileNotFoundError => e
log_result(:file_not_found, error: e)
rescue MiniMagick::Error => e
log_result(:mini_magick_error, error: e)
rescue ArgumentError, StandardError => e
log_result(:generation_failed, error: e)
end
private
@@ -38,7 +43,11 @@ class MaterialThumbnailGenerator
def video? = content_type.start_with?('video/')
def content_type = @material.file.blob.content_type.to_s
def content_type
return nil unless @material.file.attached?
@material.file.blob.content_type.to_s
end
def image_thumbnail path
image = MiniMagick::Image.open(path)
@@ -71,16 +80,47 @@ class MaterialThumbnailGenerator
'-frames:v', '1',
'-f', 'image2',
output_path)
Rails.logger.warn("ffmpeg thumbnail failed: #{ stderr }") unless status.success?
@ffmpeg_stderr << stderr if stderr.present?
status.success?
rescue Errno::ENOENT => e
Rails.logger.warn("ffmpeg unavailable for material thumbnail: #{ e.message }")
@ffmpeg_stderr << "ffmpeg unavailable: #{ e.message }"
false
end
def attach_thumbnail image
@material.thumbnail.attach(io: File.open(image.path),
filename: 'material-thumbnail.jpg',
content_type: 'image/jpeg')
blob = nil
File.open(image.path) do |io|
blob = ActiveStorage::Blob.create_and_upload!(
io:,
filename: 'material-thumbnail.jpg',
content_type: 'image/jpeg')
end
@material.thumbnail.attach(blob)
log_result(:attached)
rescue StandardError => e
blob&.purge_later
log_result(:attach_failed, error: e)
end
def log_result result, error: nil
log_payload = { material_id: @material.id,
file_blob_id: file_blob_id,
content_type:,
result:,
error_class: error&.class&.name,
error_message: error&.message,
ffmpeg_stderr: @ffmpeg_stderr.join("\n").presence }
if [:attached, :no_file, :unsupported_content_type].include?(result)
Rails.logger.info("Material thumbnail generation: #{ log_payload.to_json }")
else
Rails.logger.warn("Material thumbnail generation: #{ log_payload.to_json }")
end
result
end
def file_blob_id
return nil unless @material.file.attached?
@material.file.blob.id
end
end