このコミットが含まれているのは:
2026-06-26 00:49:18 +09:00
コミット daf9e7e6fa
12個のファイルの変更368行の追加46行の削除
+5
ファイルの表示
@@ -35,6 +35,11 @@ class MaterialVersionsController < ApplicationController
tag_name: row.tag_name,
tag_category: row.tag_category,
url: row.url,
source_kind: row.source_kind,
source_uri: row.source_uri,
source_path: row.source_path,
source_file_id: row.source_file_id,
normalized_source_key: row.normalized_source_key,
file_blob_id: row.file_blob_id,
file_filename: row.file_filename,
file_content_type: row.file_content_type,
+4 -2
ファイルの表示
@@ -1,11 +1,13 @@
class MaterialSyncSource < ApplicationRecord
SOURCE_KINDS = ['uri', 'google_drive_path', 'google_drive_file', 'legacy_drive_path'].freeze
belongs_to :created_by_user, class_name: 'User', optional: true
belongs_to :updated_by_user, class_name: 'User', optional: true
validates :name, presence: true
validates :source_kind,
presence: true,
inclusion: { in: MaterialSyncSuppression::SOURCE_KINDS }
inclusion: { in: SOURCE_KINDS }
validates :profile, presence: true, inclusion: { in: MaterialExportItem::VALID_PROFILES }
validate :source_value_must_be_present
@@ -33,7 +35,7 @@ class MaterialSyncSource < ApplicationRecord
when 'google_drive_file'
source_file_id.present? || source_uri.present?
when 'google_drive_path'
source_file_id.present? || source_uri.present? || source_path.present?
source_file_id.present? || source_uri.present?
when 'legacy_drive_path'
source_path.present?
else
+26 -9
ファイルの表示
@@ -11,6 +11,7 @@ module GoogleDrive
DRIVE_ENDPOINT = 'https://www.googleapis.com/drive/v3'
TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token'
FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
NATIVE_FILE_MIME_TYPE_PREFIX = 'application/vnd.google-apps.'
DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.readonly'
def initialize service_account_email: ENV['GOOGLE_DRIVE_SERVICE_ACCOUNT_EMAIL'],
@@ -27,6 +28,7 @@ module GoogleDrive
files = []
walk_folder(folder_id, nil) do |entry, relative_path|
next if entry['mimeType'] == FOLDER_MIME_TYPE
next if native_file?(entry['mimeType'])
files << build_file_entry(entry, relative_path)
end
@@ -35,6 +37,8 @@ module GoogleDrive
def fetch_material_file file_id
metadata = get_file(file_id)
return nil if native_file?(metadata['mimeType'])
build_file_entry(metadata, metadata['name'])
end
@@ -85,6 +89,7 @@ module GoogleDrive
name: entry['name'],
mime_type: entry['mimeType'],
relative_path: MaterialSyncExportPath.normalize_path(relative_path),
sha256_checksum: entry['sha256Checksum'],
web_view_link: entry['webViewLink'],
web_content_link: entry['webContentLink'] }
end
@@ -97,7 +102,8 @@ module GoogleDrive
response =
request_json('/files', {
q: "'#{ folder_id }' in parents and trashed = false",
fields: 'nextPageToken,files(id,name,mimeType,webViewLink,webContentLink)',
fields: 'nextPageToken,files(id,name,mimeType,sha256Checksum,' \
'webViewLink,webContentLink)',
orderBy: 'folder,name',
pageSize: 1000,
supportsAllDrives: true,
@@ -113,7 +119,7 @@ module GoogleDrive
def get_file file_id
request_json("/files/#{ file_id }",
fields: 'id,name,mimeType,webViewLink,webContentLink',
fields: 'id,name,mimeType,sha256Checksum,webViewLink,webContentLink',
supportsAllDrives: true)
end
@@ -127,13 +133,14 @@ module GoogleDrive
end
def request_binary path, params = {}
response = request(:get, path, params:)
unless response.is_a?(Net::HTTPSuccess)
raise "Google Drive download error: #{ response.code } #{ response.body }"
end
request(:get, path, params:) do |response|
unless response.is_a?(Net::HTTPSuccess)
raise "Google Drive download error: #{ response.code } #{ response.body }"
end
response.read_body do |chunk|
yield chunk
response.read_body do |chunk|
yield chunk
end
end
end
@@ -156,7 +163,13 @@ module GoogleDrive
use_ssl: true,
open_timeout: 10,
read_timeout: 60) do |http|
http.request(request)
if block_given?
http.request(request) do |response|
return yield response
end
else
http.request(request)
end
end
end
@@ -205,5 +218,9 @@ module GoogleDrive
OpenSSL::PKey::RSA.new(raw)
end
def native_file? mime_type
mime_type.to_s.start_with?(NATIVE_FILE_MIME_TYPE_PREFIX)
end
end
end
+112 -16
ファイルの表示
@@ -28,6 +28,13 @@ class MaterialSyncImporter
Material
.unscoped
.find_or_initialize_by(normalized_source_key: key)
if unchanged?(material, source)
return Result.new(material:,
action: :unchanged,
suppressed: false,
suppression: nil)
end
event_type =
if material.new_record?
:create
@@ -43,7 +50,7 @@ class MaterialSyncImporter
MaterialVersionRecorder.ensure_snapshot!(material,
created_by_user: @attributes[:updated_by_user])
end
material.assign_attributes(material_attributes.merge(source))
material.assign_attributes(material_attributes_for(material).merge(source))
material.discarded_at = nil if material.respond_to?(:discarded_at=)
material.file.attach(uploaded_blob) if uploaded_blob
material.save!
@@ -81,10 +88,28 @@ class MaterialSyncImporter
source_file_id: source[:source_file_id])
end
def material_attributes
@attributes.except(:source_kind, :source_uri, :source_path, :source_file_id,
:file_blob, :file_tempfile, :filename, :content_type,
:file_sha256, :export_path, :profile)
def material_attributes_for material
attrs =
@attributes
.except(:source_kind, :source_uri, :source_path, :source_file_id,
:file_blob, :file_tempfile, :file_downloader,
:filename, :content_type,
:file_sha256, :export_path, :profile, :tag, :url)
if material.new_record?
attrs[:tag] = @attributes[:tag] if @attributes.key?(:tag)
attrs[:url] = @attributes[:url] if @attributes.key?(:url)
return attrs
end
if @attributes.key?(:tag) && @attributes[:tag].present?
attrs[:tag] = @attributes[:tag]
end
if @attributes.key?(:url) && @attributes[:url].present?
attrs[:url] = @attributes[:url]
end
attrs
end
def suppression_for source
@@ -100,7 +125,7 @@ class MaterialSyncImporter
def uploaded_blob!
return @attributes[:file_blob] if @attributes[:file_blob]
tempfile = @attributes[:file_tempfile]
tempfile = upload_tempfile
return nil unless tempfile
tempfile.rewind
@@ -115,18 +140,22 @@ class MaterialSyncImporter
blob
end
def upload_tempfile
return @upload_tempfile if defined?(@upload_tempfile)
@upload_tempfile = @attributes[:file_tempfile]
if @upload_tempfile.blank? && @attributes[:file_downloader]
@upload_tempfile = @attributes[:file_downloader].call
end
@upload_tempfile
end
def upsert_export_item! material
export_path = @attributes[:export_path].to_s.strip
export_path = resolved_export_path(material)
return if export_path.blank?
profile = @attributes[:profile].presence || 'legacy_drive'
path = export_path
if export_path_taken_by_other?(material, profile, path)
path = MaterialSyncExportPath.uniquify(path, @attributes[:source_file_id])
end
item = material.material_export_items.find_or_initialize_by(profile:)
item.export_path = path
item = material.material_export_items.find_or_initialize_by(profile: effective_profile)
item.export_path = export_path
item.enabled = true
item.created_by_user ||= @attributes[:created_by_user]
item.save!
@@ -140,11 +169,78 @@ class MaterialSyncImporter
end
def close_tempfile!
tempfile = @attributes[:file_tempfile]
tempfile = upload_tempfile
return unless tempfile
tempfile.close! unless tempfile.closed?
rescue StandardError
nil
end
def unchanged? material, source
return false if material.new_record? || material.discarded?
return false unless same_source_attributes?(material, source)
return false unless same_export_path?(material)
return false unless same_material_attributes?(material)
same_file_snapshot?(material)
end
def same_source_attributes? material, source
material.source_kind == source[:source_kind] \
&& material.source_uri == source[:source_uri] \
&& material.source_path == source[:source_path] \
&& material.source_file_id == source[:source_file_id]
end
def same_export_path? material
expected = resolved_export_path(material)
current = material.material_export_items.find { |item| item.profile == effective_profile }
return current.blank? if expected.blank?
current&.enabled && current.export_path == expected
end
def same_material_attributes? material
same_tag_attribute?(material) && same_url_attribute?(material)
end
def same_file_snapshot? material
expected_sha256 = @attributes[:file_sha256].to_s.presence
if expected_sha256.present?
return false unless material.file.attached?
return material.file.blob.metadata['sha256'] == expected_sha256
end
!material.file.attached?
end
def effective_profile
@attributes[:profile].presence || 'legacy_drive'
end
def resolved_export_path material
export_path = @attributes[:export_path].to_s.strip
return export_path if export_path.blank?
return export_path unless export_path_taken_by_other?(material, effective_profile, export_path)
MaterialSyncExportPath.uniquify(export_path, @attributes[:source_file_id])
end
def same_tag_attribute? material
return true unless @attributes.key?(:tag)
return true if @attributes[:tag].blank?
material.tag_id == @attributes[:tag].id
end
def same_url_attribute? material
return true unless @attributes.key?(:url)
return true if @attributes[:url].blank?
material.url == @attributes[:url]
end
end
+22 -14
ファイルの表示
@@ -1,13 +1,13 @@
require 'digest'
class MaterialSyncRunner
Result = Struct.new(:imported, :updated, :suppressed, :failed, :errors, keyword_init: true)
Result = Struct.new(:imported, :updated, :unchanged, :suppressed, :failed,
:errors, keyword_init: true)
def self.sync_enabled!
results = MaterialSyncSource.enabled.order(:id).map { |source| new(source).sync! }
Result.new(imported: results.sum(&:imported),
updated: results.sum(&:updated),
unchanged: results.sum(&:unchanged),
suppressed: results.sum(&:suppressed),
failed: results.sum(&:failed),
errors: results.flat_map(&:errors))
@@ -18,9 +18,12 @@ class MaterialSyncRunner
end
def sync!
result = Result.new(imported: 0, updated: 0, suppressed: 0, failed: 0, errors: [])
result = Result.new(imported: 0, updated: 0, unchanged: 0,
suppressed: 0, failed: 0, errors: [])
candidates.each do |candidate|
next if candidate.blank?
sync_candidate!(candidate, result)
end
@@ -108,8 +111,10 @@ class MaterialSyncRunner
end
def google_drive_file_candidate
build_google_drive_candidate(drive_client.fetch_material_file(google_drive_file_id),
single_file: true)
entry = drive_client.fetch_material_file(google_drive_file_id)
return nil unless entry
build_google_drive_candidate(entry, single_file: true)
end
def build_google_drive_candidate entry, single_file: false
@@ -124,10 +129,6 @@ class MaterialSyncRunner
relative_path:,
filename: entry[:name],
source_file_id: entry[:id])
tempfile =
drive_client.download_to_tempfile(entry[:id], filename: entry[:name])
file_sha256 = Digest::SHA256.file(tempfile.path).hexdigest
tempfile.rewind
{ source_kind: 'google_drive_file',
source_uri: entry[:web_view_link] || google_drive_file_url(entry[:id]),
@@ -135,8 +136,10 @@ class MaterialSyncRunner
source_file_id: entry[:id],
filename: entry[:name],
content_type: entry[:mime_type],
file_tempfile: tempfile,
file_sha256:,
file_downloader: lambda {
drive_client.download_to_tempfile(entry[:id], filename: entry[:name])
},
file_sha256: entry[:sha256_checksum],
export_path:,
profile: @source.profile,
tag: nil,
@@ -171,13 +174,17 @@ class MaterialSyncRunner
def google_drive_folder_id
google_drive_target_id.tap do |id|
raise ArgumentError, 'google_drive_path source_file_id or source_uri is required' if id.blank?
if id.blank?
raise ArgumentError, 'google_drive_path source_file_id or source_uri is required'
end
end
end
def google_drive_file_id
google_drive_target_id.tap do |id|
raise ArgumentError, 'google_drive_file source_file_id or source_uri is required' if id.blank?
if id.blank?
raise ArgumentError, 'google_drive_file source_file_id or source_uri is required'
end
end
end
@@ -201,6 +208,7 @@ class MaterialSyncRunner
def log_result result
Rails.logger.info(material_sync_log(imported: result.imported,
updated: result.updated,
unchanged: result.unchanged,
suppressed: result.suppressed,
failed: result.failed))
end
+5
ファイルの表示
@@ -33,6 +33,11 @@ class MaterialVersionRecorder < VersionRecorder
tag: @record.tag,
tag_name: @record.tag&.name,
tag_category: @record.tag&.category,
source_kind: @record.source_kind,
source_uri: @record.source_uri,
source_path: @record.source_path,
source_file_id: @record.source_file_id,
normalized_source_key: @record.normalized_source_key,
export_paths_json: @record.snapshot_export_paths,
discarded_at: @record.discarded_at,
file_blob_id: file_snapshot[:file_blob_id],
+19
ファイルの表示
@@ -0,0 +1,19 @@
class AddSourceSnapshotToMaterialVersions < ActiveRecord::Migration[8.0]
def change
unless column_exists?(:material_versions, :source_kind)
add_column :material_versions, :source_kind, :string
end
unless column_exists?(:material_versions, :source_uri)
add_column :material_versions, :source_uri, :string
end
unless column_exists?(:material_versions, :source_path)
add_column :material_versions, :source_path, :string
end
unless column_exists?(:material_versions, :source_file_id)
add_column :material_versions, :source_file_id, :string
end
unless column_exists?(:material_versions, :normalized_source_key)
add_column :material_versions, :normalized_source_key, :string
end
end
end
+159
ファイルの表示
@@ -0,0 +1,159 @@
class BackfillInitialMaterialVersions < ActiveRecord::Migration[8.0]
class MigrationMaterial < ActiveRecord::Base
self.table_name = 'materials'
end
class MigrationMaterialExportItem < ActiveRecord::Base
self.table_name = 'material_export_items'
end
class MigrationMaterialVersion < ActiveRecord::Base
self.table_name = 'material_versions'
end
class MigrationStorageAttachment < ActiveRecord::Base
self.table_name = 'active_storage_attachments'
end
class MigrationStorageBlob < ActiveRecord::Base
self.table_name = 'active_storage_blobs'
end
class MigrationTag < ActiveRecord::Base
self.table_name = 'tags'
end
class MigrationTagName < ActiveRecord::Base
self.table_name = 'tag_names'
end
def up
say_with_time 'Backfilling initial material versions' do
materials_without_versions.find_in_batches(batch_size: 500) do |materials|
material_ids = materials.map(&:id)
tags_by_id = load_tags(materials.map(&:tag_id).compact.uniq)
export_paths_by_material_id = load_export_paths(material_ids)
file_snapshots_by_material_id = load_file_snapshots(material_ids)
rows = materials.map do |material|
tag = tags_by_id[material.tag_id]
file_snapshot = file_snapshots_by_material_id[material.id] || empty_file_snapshot
{ material_id: material.id,
version_no: 1,
event_type: 'create',
url: material.url,
parent_id: material.parent_id,
tag_id: material.tag_id,
tag_name: tag&.fetch(:name, nil),
tag_category: tag&.fetch(:category, nil),
source_kind: material.source_kind,
source_uri: material.source_uri,
source_path: material.source_path,
source_file_id: material.source_file_id,
normalized_source_key: material.normalized_source_key,
export_paths_json: export_paths_by_material_id[material.id] || { },
discarded_at: material.discarded_at,
file_blob_id: file_snapshot[:file_blob_id],
file_filename: file_snapshot[:file_filename],
file_content_type: file_snapshot[:file_content_type],
file_byte_size: file_snapshot[:file_byte_size],
file_checksum: file_snapshot[:file_checksum],
file_sha256: file_snapshot[:file_sha256],
created_by_user_id: material.created_by_user_id,
updated_by_user_id: material.updated_by_user_id,
created_at: material.created_at,
updated_at: material.created_at }
end
MigrationMaterialVersion.insert_all!(rows) if rows.present?
end
end
execute <<~SQL.squish
UPDATE materials
SET version_no = COALESCE(
(
SELECT MAX(material_versions.version_no)
FROM material_versions
WHERE material_versions.material_id = materials.id
),
1
)
SQL
end
def down
raise ActiveRecord::IrreversibleMigration
end
private
def materials_without_versions
MigrationMaterial.where.not(id: MigrationMaterialVersion.select(:material_id))
end
def load_tags tag_ids
return { } if tag_ids.empty?
tag_name_ids_by_tag_id = MigrationTag.where(id: tag_ids).pluck(:id, :tag_name_id).to_h
tag_names_by_id =
MigrationTagName
.where(id: tag_name_ids_by_tag_id.values.uniq)
.pluck(:id, :name)
.to_h
categories_by_tag_id = MigrationTag.where(id: tag_ids).pluck(:id, :category).to_h
tag_ids.each_with_object({ }) do |tag_id, hash|
hash[tag_id] = { name: tag_names_by_id[tag_name_ids_by_tag_id[tag_id]],
category: categories_by_tag_id[tag_id] }
end
end
def load_export_paths material_ids
MigrationMaterialExportItem
.where(material_id: material_ids)
.order(:material_id, :profile)
.pluck(:material_id, :profile, :export_path)
.each_with_object({ }) do |(material_id, profile, export_path), hash|
hash[material_id] ||= { }
hash[material_id][profile] = export_path
end
end
def load_file_snapshots material_ids
attachments_by_material_id =
MigrationStorageAttachment
.where(record_type: 'Material', name: 'file', record_id: material_ids)
.pluck(:record_id, :blob_id)
.to_h
blobs_by_id =
MigrationStorageBlob
.where(id: attachments_by_material_id.values.uniq)
.index_by(&:id)
attachments_by_material_id.each_with_object({ }) do |(material_id, blob_id), hash|
blob = blobs_by_id[blob_id]
next unless blob
hash[material_id] = {
file_blob_id: blob.id,
file_filename: blob.filename.to_s,
file_content_type: blob.content_type,
file_byte_size: blob.byte_size,
file_checksum: blob.checksum,
file_sha256: blob.metadata['sha256']
}
end
end
def empty_file_snapshot
{ file_blob_id: nil,
file_filename: nil,
file_content_type: nil,
file_byte_size: nil,
file_checksum: nil,
file_sha256: nil }
end
end
生成ファイル
+6 -1
ファイルの表示
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.0].define(version: 2026_06_25_010000) do
ActiveRecord::Schema[8.0].define(version: 2026_06_26_010000) do
create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
t.string "name", null: false
t.string "record_type", null: false
@@ -213,6 +213,11 @@ ActiveRecord::Schema[8.0].define(version: 2026_06_25_010000) do
t.bigint "file_byte_size"
t.string "file_checksum"
t.string "file_sha256"
t.string "source_kind"
t.string "source_uri"
t.string "source_path"
t.string "source_file_id"
t.string "normalized_source_key"
t.index ["created_by_user_id"], name: "index_material_versions_on_created_by_user_id"
t.index ["discarded_at"], name: "index_material_versions_on_discarded_at"
t.index ["file_blob_id"], name: "index_material_versions_on_file_blob_id"
+4 -4
ファイルの表示
@@ -9,19 +9,19 @@
# end
material_sync_source_uri = ENV['MATERIAL_SYNC_SOURCE_URI']
material_sync_source_tag_name = ENV['MATERIAL_SYNC_SOURCE_TAG_NAME']
material_sync_source_file_id = ENV['MATERIAL_SYNC_SOURCE_FILE_ID']
if material_sync_source_uri.present? && material_sync_source_tag_name.present?
if material_sync_source_uri.present? || material_sync_source_file_id.present?
source = MaterialSyncSource.find_or_initialize_by(
name: ENV.fetch('MATERIAL_SYNC_SOURCE_NAME', 'Legacy URI material sync'))
source.assign_attributes(
source_kind: ENV.fetch('MATERIAL_SYNC_SOURCE_KIND', 'uri'),
source_uri: material_sync_source_uri,
source_path: ENV['MATERIAL_SYNC_SOURCE_PATH'],
source_file_id: ENV['MATERIAL_SYNC_SOURCE_FILE_ID'],
source_file_id: material_sync_source_file_id,
profile: ENV.fetch('MATERIAL_SYNC_SOURCE_PROFILE', 'legacy_drive'),
enabled: ENV.fetch('MATERIAL_SYNC_SOURCE_ENABLED', 'true') != 'false',
default_tag_name: material_sync_source_tag_name,
default_tag_name: ENV['MATERIAL_SYNC_SOURCE_TAG_NAME'],
export_path_prefix: ENV['MATERIAL_SYNC_EXPORT_PATH_PREFIX'])
source.save!
end
+1
ファイルの表示
@@ -5,6 +5,7 @@ namespace :materials do
message = [
"materials:sync imported=#{ result.imported }",
"updated=#{ result.updated }",
"unchanged=#{ result.unchanged }",
"suppressed=#{ result.suppressed }",
"failed=#{ result.failed }"
].join(' ')
+5
ファイルの表示
@@ -179,6 +179,11 @@ export type MaterialVersion = {
tagName: string | null
tagCategory: Category | null
url: string | null
sourceKind: string | null
sourceUri: string | null
sourcePath: string | null
sourceFileId: string | null
normalizedSourceKey: string | null
fileBlobId: number | null
fileFilename: string | null
fileContentType: string | null