Merge remote-tracking branch 'origin/main' into feature/351

2026-07-02 01:55:35 +09:00
@@ -0,0 +1,88 @@
+class EnhanceMaterialManagement < ActiveRecord::Migration[8.0]
+  def up
+    change_table :materials, bulk: true do |t|
+      t.integer :version_no, null: false, default: 1
+    end
+
+    change_table :material_versions, bulk: true do |t|
+      t.string :event_type
+      t.string :tag_name
+      t.string :tag_category
+      t.json :export_paths_json
+      t.bigint :file_blob_id
+      t.string :file_filename
+      t.string :file_content_type
+      t.bigint :file_byte_size
+      t.string :file_checksum
+      t.string :file_sha256
+    end
+
+    execute <<~SQL.squish
+      UPDATE material_versions
+      SET event_type = CASE
+                       WHEN version_no = 1 THEN 'create'
+                       ELSE 'update'
+                       END
+      WHERE event_type IS NULL
+    SQL
+
+    change_column_null :material_versions, :event_type, false
+    add_index :material_versions, :file_blob_id
+
+    add_check_constraint :material_versions,
+                         "event_type IN ('create', 'update', 'discard', 'restore')",
+                         name: 'material_versions_event_type_valid'
+
+    create_table :material_export_items do |t|
+      t.references :material, null: false, foreign_key: true
+      t.string :profile, null: false, default: 'legacy_drive'
+      t.string :export_path, null: false
+      t.boolean :enabled, null: false, default: true
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+
+      t.index [:profile, :export_path], unique: true
+      t.index [:material_id, :profile], unique: true
+    end
+
+    create_table :material_import_blocks do |t|
+      t.string :match_kind, null: false
+      t.string :sha256
+      t.string :external_path_pattern
+      t.string :reason, null: false
+      t.text :note
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+    end
+
+    execute <<~SQL.squish
+      UPDATE materials
+      SET version_no = COALESCE(
+        (SELECT MAX(material_versions.version_no)
+         FROM material_versions
+         WHERE material_versions.material_id = materials.id),
+        1)
+    SQL
+  end
+
+  def down
+    drop_table :material_import_blocks
+    drop_table :material_export_items
+
+    remove_check_constraint :material_versions, name: 'material_versions_event_type_valid'
+
+    remove_index :material_versions, :file_blob_id
+    remove_column :material_versions, :event_type
+    remove_column :material_versions, :tag_name
+    remove_column :material_versions, :tag_category
+    remove_column :material_versions, :export_paths_json
+    remove_column :material_versions, :file_blob_id
+    remove_column :material_versions, :file_filename
+    remove_column :material_versions, :file_content_type
+    remove_column :material_versions, :file_byte_size
+    remove_column :material_versions, :file_checksum
+    remove_column :material_versions, :file_sha256
+
+    remove_column :materials, :version_no
+  end
+end
@@ -0,0 +1,118 @@
+class ReconcileMaterialSyncSuppressions < ActiveRecord::Migration[8.0]
+  MATERIAL_VERSION_EVENT_CONSTRAINT = 'material_versions_event_type_valid'
+
+  def up
+    ensure_material_sync_suppressions!
+    ensure_material_source_columns!
+    ensure_material_active_source_key!
+    remove_material_file_suppression!
+    remove_material_version_file_suppression!
+    reconcile_material_version_event_constraint!
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+
+  private
+
+  def ensure_material_sync_suppressions!
+    return if table_exists?(:material_sync_suppressions)
+
+    create_table :material_sync_suppressions do |t|
+      t.string :source_kind, null: false
+      t.string :source_uri
+      t.string :drive_path
+      t.string :drive_file_id
+      t.string :normalized_source_key, null: false
+      t.string :reason, null: false
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+
+      t.index :normalized_source_key, unique: true
+      t.index :source_kind
+      t.index :drive_file_id
+    end
+  end
+
+  def ensure_material_source_columns!
+    add_column :materials, :source_kind, :string unless column_exists?(:materials, :source_kind)
+    add_column :materials, :source_uri, :string unless column_exists?(:materials, :source_uri)
+    add_column :materials, :source_path, :string unless column_exists?(:materials, :source_path)
+    unless column_exists?(:materials, :source_file_id)
+      add_column :materials, :source_file_id, :string
+    end
+    unless column_exists?(:materials, :normalized_source_key)
+      add_column :materials, :normalized_source_key, :string
+    end
+
+    if index_exists?(:materials, :normalized_source_key)
+      remove_index :materials, column: :normalized_source_key
+    end
+  end
+
+  def ensure_material_active_source_key!
+    unless column_exists?(:materials, :active_normalized_source_key)
+      change_table :materials do |t|
+        t.virtual :active_normalized_source_key,
+                  type: :string,
+                  as: 'IF(discarded_at IS NULL, normalized_source_key, NULL)',
+                  stored: false
+      end
+    end
+
+    return if index_exists?(:materials,
+                            :active_normalized_source_key,
+                            name: 'index_materials_on_active_normalized_source_key')
+
+    add_index :materials,
+              :active_normalized_source_key,
+              unique: true,
+              name: 'index_materials_on_active_normalized_source_key'
+  end
+
+  def remove_material_file_suppression!
+    if foreign_key_exists?(:materials, :users, column: :file_suppressed_by_user_id)
+      remove_foreign_key :materials, column: :file_suppressed_by_user_id
+    end
+
+    if index_exists?(:materials,
+                    :file_suppressed_by_user_id,
+                    name: 'index_materials_on_file_suppressed_by_user_id')
+      remove_index :materials, name: 'index_materials_on_file_suppressed_by_user_id'
+    end
+
+    remove_column :materials, :file_suppressed_at if column_exists?(
+        :materials,
+        :file_suppressed_at)
+    remove_column :materials, :file_suppressed_by_user_id if column_exists?(
+        :materials,
+        :file_suppressed_by_user_id)
+    remove_column :materials, :file_suppression_reason if column_exists?(
+        :materials,
+        :file_suppression_reason)
+  end
+
+  def remove_material_version_file_suppression!
+    remove_column :material_versions, :file_suppressed_at if column_exists?(
+        :material_versions,
+        :file_suppressed_at)
+    remove_column :material_versions, :file_suppression_reason if column_exists?(
+        :material_versions,
+        :file_suppression_reason)
+  end
+
+  def reconcile_material_version_event_constraint!
+    constraint = check_constraints(:material_versions).find do |candidate|
+      candidate.name == MATERIAL_VERSION_EVENT_CONSTRAINT
+    end
+    return if constraint && !constraint.expression.include?('suppress')
+
+    if constraint
+      remove_check_constraint :material_versions, name: MATERIAL_VERSION_EVENT_CONSTRAINT
+    end
+    add_check_constraint :material_versions,
+                         "event_type IN ('create', 'update', 'discard', 'restore')",
+                         name: MATERIAL_VERSION_EVENT_CONSTRAINT
+  end
+end
@@ -0,0 +1,22 @@
+class CreateMaterialSyncSources < ActiveRecord::Migration[8.0]
+  def change
+    create_table :material_sync_sources do |t|
+      t.string :name, null: false
+      t.string :source_kind, null: false
+      t.string :source_uri
+      t.string :source_path
+      t.string :source_file_id
+      t.string :profile, null: false, default: 'legacy_drive'
+      t.boolean :enabled, null: false, default: true
+      t.string :default_tag_name
+      t.string :export_path_prefix
+      t.datetime :last_synced_at
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.references :updated_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+
+      t.index :enabled
+      t.index :source_kind
+    end
+  end
+end
@@ -0,0 +1,19 @@
+class AddSourceSnapshotToMaterialVersions < ActiveRecord::Migration[8.0]
+  def change
+    unless column_exists?(:material_versions, :source_kind)
+      add_column :material_versions, :source_kind, :string
+    end
+    unless column_exists?(:material_versions, :source_uri)
+      add_column :material_versions, :source_uri, :string
+    end
+    unless column_exists?(:material_versions, :source_path)
+      add_column :material_versions, :source_path, :string
+    end
+    unless column_exists?(:material_versions, :source_file_id)
+      add_column :material_versions, :source_file_id, :string
+    end
+    unless column_exists?(:material_versions, :normalized_source_key)
+      add_column :material_versions, :normalized_source_key, :string
+    end
+  end
+end
@@ -0,0 +1,171 @@
+require 'json'
+
+class BackfillInitialMaterialVersions < ActiveRecord::Migration[8.0]
+  class MigrationMaterial < ActiveRecord::Base
+    self.table_name = 'materials'
+  end
+
+  class MigrationMaterialExportItem < ActiveRecord::Base
+    self.table_name = 'material_export_items'
+  end
+
+  class MigrationMaterialVersion < ActiveRecord::Base
+    self.table_name = 'material_versions'
+  end
+
+  class MigrationStorageAttachment < ActiveRecord::Base
+    self.table_name = 'active_storage_attachments'
+  end
+
+  class MigrationStorageBlob < ActiveRecord::Base
+    self.table_name = 'active_storage_blobs'
+  end
+
+  class MigrationTag < ActiveRecord::Base
+    self.table_name = 'tags'
+  end
+
+  class MigrationTagName < ActiveRecord::Base
+    self.table_name = 'tag_names'
+  end
+
+  def up
+    say_with_time 'Backfilling initial material versions' do
+      materials_without_versions.find_in_batches(batch_size: 500) do |materials|
+        material_ids = materials.map(&:id)
+        tags_by_id = load_tags(materials.map(&:tag_id).compact.uniq)
+        export_paths_by_material_id = load_export_paths(material_ids)
+        file_snapshots_by_material_id = load_file_snapshots(material_ids)
+
+        rows = materials.map do |material|
+          tag = tags_by_id[material.tag_id]
+          file_snapshot = file_snapshots_by_material_id[material.id] || empty_file_snapshot
+
+          { material_id: material.id,
+            version_no: 1,
+            event_type: 'create',
+            url: material.url,
+            parent_id: material.parent_id,
+            tag_id: material.tag_id,
+            tag_name: tag&.fetch(:name, nil),
+            tag_category: tag&.fetch(:category, nil),
+            source_kind: material.source_kind,
+            source_uri: material.source_uri,
+            source_path: material.source_path,
+            source_file_id: material.source_file_id,
+            normalized_source_key: material.normalized_source_key,
+            export_paths_json: export_paths_by_material_id[material.id] || { },
+            discarded_at: material.discarded_at,
+            file_blob_id: file_snapshot[:file_blob_id],
+            file_filename: file_snapshot[:file_filename],
+            file_content_type: file_snapshot[:file_content_type],
+            file_byte_size: file_snapshot[:file_byte_size],
+            file_checksum: file_snapshot[:file_checksum],
+            file_sha256: file_snapshot[:file_sha256],
+            created_by_user_id: material.created_by_user_id,
+            updated_by_user_id: material.updated_by_user_id,
+            created_at: material.created_at,
+            updated_at: material.created_at }
+        end
+
+        MigrationMaterialVersion.insert_all!(rows) if rows.present?
+      end
+    end
+
+    execute <<~SQL.squish
+      UPDATE materials
+      SET version_no = COALESCE(
+        (
+          SELECT MAX(material_versions.version_no)
+          FROM material_versions
+          WHERE material_versions.material_id = materials.id
+        ),
+        1
+      )
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+
+  private
+
+  def materials_without_versions
+    MigrationMaterial.where.not(id: MigrationMaterialVersion.select(:material_id))
+  end
+
+  def load_tags tag_ids
+    return { } if tag_ids.empty?
+
+    tag_name_ids_by_tag_id = MigrationTag.where(id: tag_ids).pluck(:id, :tag_name_id).to_h
+    tag_names_by_id =
+        MigrationTagName
+        .where(id: tag_name_ids_by_tag_id.values.uniq)
+        .pluck(:id, :name)
+        .to_h
+    categories_by_tag_id = MigrationTag.where(id: tag_ids).pluck(:id, :category).to_h
+
+    tag_ids.each_with_object({ }) do |tag_id, hash|
+      hash[tag_id] = { name: tag_names_by_id[tag_name_ids_by_tag_id[tag_id]],
+                       category: categories_by_tag_id[tag_id] }
+    end
+  end
+
+  def load_export_paths material_ids
+    MigrationMaterialExportItem
+      .where(material_id: material_ids)
+      .order(:material_id, :profile)
+      .pluck(:material_id, :profile, :export_path)
+      .each_with_object({ }) do |(material_id, profile, export_path), hash|
+        hash[material_id] ||= { }
+        hash[material_id][profile] = export_path
+      end
+  end
+
+  def load_file_snapshots material_ids
+    attachments_by_material_id =
+        MigrationStorageAttachment
+        .where(record_type: 'Material', name: 'file', record_id: material_ids)
+        .pluck(:record_id, :blob_id)
+        .to_h
+
+    blobs_by_id =
+        MigrationStorageBlob
+        .where(id: attachments_by_material_id.values.uniq)
+        .index_by(&:id)
+
+    attachments_by_material_id.each_with_object({ }) do |(material_id, blob_id), hash|
+      blob = blobs_by_id[blob_id]
+      next unless blob
+
+      hash[material_id] = {
+        file_blob_id: blob.id,
+        file_filename: blob.filename.to_s,
+        file_content_type: blob.content_type,
+        file_byte_size: blob.byte_size,
+        file_checksum: blob.checksum,
+        file_sha256: blob_metadata(blob)['sha256']
+      }
+    end
+  end
+
+  def empty_file_snapshot
+    { file_blob_id: nil,
+      file_filename: nil,
+      file_content_type: nil,
+      file_byte_size: nil,
+      file_checksum: nil,
+      file_sha256: nil }
+  end
+
+  def blob_metadata blob
+    metadata = blob.metadata
+    return metadata if metadata.is_a?(Hash)
+    return { } if metadata.blank?
+
+    JSON.parse(metadata)
+  rescue JSON::ParserError
+    { }
+  end
+end