Merge remote-tracking branch 'origin/main' into feature/351

backend/db/migrate/20260623000000_enhance_material_management.rb

@@ -0,0 +1,88 @@
+class EnhanceMaterialManagement < ActiveRecord::Migration[8.0]
+  def up
+    change_table :materials, bulk: true do |t|
+      t.integer :version_no, null: false, default: 1
+    end
+
+    change_table :material_versions, bulk: true do |t|
+      t.string :event_type
+      t.string :tag_name
+      t.string :tag_category
+      t.json :export_paths_json
+      t.bigint :file_blob_id
+      t.string :file_filename
+      t.string :file_content_type
+      t.bigint :file_byte_size
+      t.string :file_checksum
+      t.string :file_sha256
+    end
+
+    execute <<~SQL.squish
+      UPDATE material_versions
+      SET event_type = CASE
+                       WHEN version_no = 1 THEN 'create'
+                       ELSE 'update'
+                       END
+      WHERE event_type IS NULL
+    SQL
+
+    change_column_null :material_versions, :event_type, false
+    add_index :material_versions, :file_blob_id
+
+    add_check_constraint :material_versions,
+                         "event_type IN ('create', 'update', 'discard', 'restore')",
+                         name: 'material_versions_event_type_valid'
+
+    create_table :material_export_items do |t|
+      t.references :material, null: false, foreign_key: true
+      t.string :profile, null: false, default: 'legacy_drive'
+      t.string :export_path, null: false
+      t.boolean :enabled, null: false, default: true
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+
+      t.index [:profile, :export_path], unique: true
+      t.index [:material_id, :profile], unique: true
+    end
+
+    create_table :material_import_blocks do |t|
+      t.string :match_kind, null: false
+      t.string :sha256
+      t.string :external_path_pattern
+      t.string :reason, null: false
+      t.text :note
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+    end
+
+    execute <<~SQL.squish
+      UPDATE materials
+      SET version_no = COALESCE(
+        (SELECT MAX(material_versions.version_no)
+         FROM material_versions
+         WHERE material_versions.material_id = materials.id),
+        1)
+    SQL
+  end
+
+  def down
+    drop_table :material_import_blocks
+    drop_table :material_export_items
+
+    remove_check_constraint :material_versions, name: 'material_versions_event_type_valid'
+
+    remove_index :material_versions, :file_blob_id
+    remove_column :material_versions, :event_type
+    remove_column :material_versions, :tag_name
+    remove_column :material_versions, :tag_category
+    remove_column :material_versions, :export_paths_json
+    remove_column :material_versions, :file_blob_id
+    remove_column :material_versions, :file_filename
+    remove_column :material_versions, :file_content_type
+    remove_column :material_versions, :file_byte_size
+    remove_column :material_versions, :file_checksum
+    remove_column :material_versions, :file_sha256
+
+    remove_column :materials, :version_no
+  end
+end

backend/db/migrate/20260625000000_reconcile_material_sync_suppressions.rb

@@ -0,0 +1,118 @@
+class ReconcileMaterialSyncSuppressions < ActiveRecord::Migration[8.0]
+  MATERIAL_VERSION_EVENT_CONSTRAINT = 'material_versions_event_type_valid'
+
+  def up
+    ensure_material_sync_suppressions!
+    ensure_material_source_columns!
+    ensure_material_active_source_key!
+    remove_material_file_suppression!
+    remove_material_version_file_suppression!
+    reconcile_material_version_event_constraint!
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+
+  private
+
+  def ensure_material_sync_suppressions!
+    return if table_exists?(:material_sync_suppressions)
+
+    create_table :material_sync_suppressions do |t|
+      t.string :source_kind, null: false
+      t.string :source_uri
+      t.string :drive_path
+      t.string :drive_file_id
+      t.string :normalized_source_key, null: false
+      t.string :reason, null: false
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+
+      t.index :normalized_source_key, unique: true
+      t.index :source_kind
+      t.index :drive_file_id
+    end
+  end
+
+  def ensure_material_source_columns!
+    add_column :materials, :source_kind, :string unless column_exists?(:materials, :source_kind)
+    add_column :materials, :source_uri, :string unless column_exists?(:materials, :source_uri)
+    add_column :materials, :source_path, :string unless column_exists?(:materials, :source_path)
+    unless column_exists?(:materials, :source_file_id)
+      add_column :materials, :source_file_id, :string
+    end
+    unless column_exists?(:materials, :normalized_source_key)
+      add_column :materials, :normalized_source_key, :string
+    end
+
+    if index_exists?(:materials, :normalized_source_key)
+      remove_index :materials, column: :normalized_source_key
+    end
+  end
+
+  def ensure_material_active_source_key!
+    unless column_exists?(:materials, :active_normalized_source_key)
+      change_table :materials do |t|
+        t.virtual :active_normalized_source_key,
+                  type: :string,
+                  as: 'IF(discarded_at IS NULL, normalized_source_key, NULL)',
+                  stored: false
+      end
+    end
+
+    return if index_exists?(:materials,
+                            :active_normalized_source_key,
+                            name: 'index_materials_on_active_normalized_source_key')
+
+    add_index :materials,
+              :active_normalized_source_key,
+              unique: true,
+              name: 'index_materials_on_active_normalized_source_key'
+  end
+
+  def remove_material_file_suppression!
+    if foreign_key_exists?(:materials, :users, column: :file_suppressed_by_user_id)
+      remove_foreign_key :materials, column: :file_suppressed_by_user_id
+    end
+
+    if index_exists?(:materials,
+                    :file_suppressed_by_user_id,
+                    name: 'index_materials_on_file_suppressed_by_user_id')
+      remove_index :materials, name: 'index_materials_on_file_suppressed_by_user_id'
+    end
+
+    remove_column :materials, :file_suppressed_at if column_exists?(
+        :materials,
+        :file_suppressed_at)
+    remove_column :materials, :file_suppressed_by_user_id if column_exists?(
+        :materials,
+        :file_suppressed_by_user_id)
+    remove_column :materials, :file_suppression_reason if column_exists?(
+        :materials,
+        :file_suppression_reason)
+  end
+
+  def remove_material_version_file_suppression!
+    remove_column :material_versions, :file_suppressed_at if column_exists?(
+        :material_versions,
+        :file_suppressed_at)
+    remove_column :material_versions, :file_suppression_reason if column_exists?(
+        :material_versions,
+        :file_suppression_reason)
+  end
+
+  def reconcile_material_version_event_constraint!
+    constraint = check_constraints(:material_versions).find do |candidate|
+      candidate.name == MATERIAL_VERSION_EVENT_CONSTRAINT
+    end
+    return if constraint && !constraint.expression.include?('suppress')
+
+    if constraint
+      remove_check_constraint :material_versions, name: MATERIAL_VERSION_EVENT_CONSTRAINT
+    end
+    add_check_constraint :material_versions,
+                         "event_type IN ('create', 'update', 'discard', 'restore')",
+                         name: MATERIAL_VERSION_EVENT_CONSTRAINT
+  end
+end

backend/db/migrate/20260625010000_create_material_sync_sources.rb

@@ -0,0 +1,22 @@
+class CreateMaterialSyncSources < ActiveRecord::Migration[8.0]
+  def change
+    create_table :material_sync_sources do |t|
+      t.string :name, null: false
+      t.string :source_kind, null: false
+      t.string :source_uri
+      t.string :source_path
+      t.string :source_file_id
+      t.string :profile, null: false, default: 'legacy_drive'
+      t.boolean :enabled, null: false, default: true
+      t.string :default_tag_name
+      t.string :export_path_prefix
+      t.datetime :last_synced_at
+      t.references :created_by_user, foreign_key: { to_table: :users }
+      t.references :updated_by_user, foreign_key: { to_table: :users }
+      t.timestamps
+
+      t.index :enabled
+      t.index :source_kind
+    end
+  end
+end

backend/db/migrate/20260626000000_add_source_snapshot_to_material_versions.rb

@@ -0,0 +1,19 @@
+class AddSourceSnapshotToMaterialVersions < ActiveRecord::Migration[8.0]
+  def change
+    unless column_exists?(:material_versions, :source_kind)
+      add_column :material_versions, :source_kind, :string
+    end
+    unless column_exists?(:material_versions, :source_uri)
+      add_column :material_versions, :source_uri, :string
+    end
+    unless column_exists?(:material_versions, :source_path)
+      add_column :material_versions, :source_path, :string
+    end
+    unless column_exists?(:material_versions, :source_file_id)
+      add_column :material_versions, :source_file_id, :string
+    end
+    unless column_exists?(:material_versions, :normalized_source_key)
+      add_column :material_versions, :normalized_source_key, :string
+    end
+  end
+end

backend/db/migrate/20260626010000_backfill_initial_material_versions.rb

@@ -0,0 +1,171 @@
+require 'json'
+
+class BackfillInitialMaterialVersions < ActiveRecord::Migration[8.0]
+  class MigrationMaterial < ActiveRecord::Base
+    self.table_name = 'materials'
+  end
+
+  class MigrationMaterialExportItem < ActiveRecord::Base
+    self.table_name = 'material_export_items'
+  end
+
+  class MigrationMaterialVersion < ActiveRecord::Base
+    self.table_name = 'material_versions'
+  end
+
+  class MigrationStorageAttachment < ActiveRecord::Base
+    self.table_name = 'active_storage_attachments'
+  end
+
+  class MigrationStorageBlob < ActiveRecord::Base
+    self.table_name = 'active_storage_blobs'
+  end
+
+  class MigrationTag < ActiveRecord::Base
+    self.table_name = 'tags'
+  end
+
+  class MigrationTagName < ActiveRecord::Base
+    self.table_name = 'tag_names'
+  end
+
+  def up
+    say_with_time 'Backfilling initial material versions' do
+      materials_without_versions.find_in_batches(batch_size: 500) do |materials|
+        material_ids = materials.map(&:id)
+        tags_by_id = load_tags(materials.map(&:tag_id).compact.uniq)
+        export_paths_by_material_id = load_export_paths(material_ids)
+        file_snapshots_by_material_id = load_file_snapshots(material_ids)
+
+        rows = materials.map do |material|
+          tag = tags_by_id[material.tag_id]
+          file_snapshot = file_snapshots_by_material_id[material.id] || empty_file_snapshot
+
+          { material_id: material.id,
+            version_no: 1,
+            event_type: 'create',
+            url: material.url,
+            parent_id: material.parent_id,
+            tag_id: material.tag_id,
+            tag_name: tag&.fetch(:name, nil),
+            tag_category: tag&.fetch(:category, nil),
+            source_kind: material.source_kind,
+            source_uri: material.source_uri,
+            source_path: material.source_path,
+            source_file_id: material.source_file_id,
+            normalized_source_key: material.normalized_source_key,
+            export_paths_json: export_paths_by_material_id[material.id] || { },
+            discarded_at: material.discarded_at,
+            file_blob_id: file_snapshot[:file_blob_id],
+            file_filename: file_snapshot[:file_filename],
+            file_content_type: file_snapshot[:file_content_type],
+            file_byte_size: file_snapshot[:file_byte_size],
+            file_checksum: file_snapshot[:file_checksum],
+            file_sha256: file_snapshot[:file_sha256],
+            created_by_user_id: material.created_by_user_id,
+            updated_by_user_id: material.updated_by_user_id,
+            created_at: material.created_at,
+            updated_at: material.created_at }
+        end
+
+        MigrationMaterialVersion.insert_all!(rows) if rows.present?
+      end
+    end
+
+    execute <<~SQL.squish
+      UPDATE materials
+      SET version_no = COALESCE(
+        (
+          SELECT MAX(material_versions.version_no)
+          FROM material_versions
+          WHERE material_versions.material_id = materials.id
+        ),
+        1
+      )
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+
+  private
+
+  def materials_without_versions
+    MigrationMaterial.where.not(id: MigrationMaterialVersion.select(:material_id))
+  end
+
+  def load_tags tag_ids
+    return { } if tag_ids.empty?
+
+    tag_name_ids_by_tag_id = MigrationTag.where(id: tag_ids).pluck(:id, :tag_name_id).to_h
+    tag_names_by_id =
+        MigrationTagName
+        .where(id: tag_name_ids_by_tag_id.values.uniq)
+        .pluck(:id, :name)
+        .to_h
+    categories_by_tag_id = MigrationTag.where(id: tag_ids).pluck(:id, :category).to_h
+
+    tag_ids.each_with_object({ }) do |tag_id, hash|
+      hash[tag_id] = { name: tag_names_by_id[tag_name_ids_by_tag_id[tag_id]],
+                       category: categories_by_tag_id[tag_id] }
+    end
+  end
+
+  def load_export_paths material_ids
+    MigrationMaterialExportItem
+      .where(material_id: material_ids)
+      .order(:material_id, :profile)
+      .pluck(:material_id, :profile, :export_path)
+      .each_with_object({ }) do |(material_id, profile, export_path), hash|
+        hash[material_id] ||= { }
+        hash[material_id][profile] = export_path
+      end
+  end
+
+  def load_file_snapshots material_ids
+    attachments_by_material_id =
+        MigrationStorageAttachment
+        .where(record_type: 'Material', name: 'file', record_id: material_ids)
+        .pluck(:record_id, :blob_id)
+        .to_h
+
+    blobs_by_id =
+        MigrationStorageBlob
+        .where(id: attachments_by_material_id.values.uniq)
+        .index_by(&:id)
+
+    attachments_by_material_id.each_with_object({ }) do |(material_id, blob_id), hash|
+      blob = blobs_by_id[blob_id]
+      next unless blob
+
+      hash[material_id] = {
+        file_blob_id: blob.id,
+        file_filename: blob.filename.to_s,
+        file_content_type: blob.content_type,
+        file_byte_size: blob.byte_size,
+        file_checksum: blob.checksum,
+        file_sha256: blob_metadata(blob)['sha256']
+      }
+    end
+  end
+
+  def empty_file_snapshot
+    { file_blob_id: nil,
+      file_filename: nil,
+      file_content_type: nil,
+      file_byte_size: nil,
+      file_checksum: nil,
+      file_sha256: nil }
+  end
+
+  def blob_metadata blob
+    metadata = blob.metadata
+    return metadata if metadata.is_a?(Hash)
+    return { } if metadata.blank?
+
+    JSON.parse(metadata)
+  rescue JSON::ParserError
+    { }
+  end
+end

backend/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2026_06_22_020000) do
+ActiveRecord::Schema[8.0].define(version: 2026_06_26_010000) do
   create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.string "name", null: false
     t.string "record_type", null: false
@@ -130,6 +130,69 @@ ActiveRecord::Schema[8.0].define(version: 2026_06_22_020000) do
     t.index ["ip_address"], name: "index_ip_addresses_on_ip_address", unique: true
   end
 
+  create_table "material_export_items", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
+    t.bigint "material_id", null: false
+    t.string "profile", default: "legacy_drive", null: false
+    t.string "export_path", null: false
+    t.boolean "enabled", default: true, null: false
+    t.bigint "created_by_user_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["created_by_user_id"], name: "index_material_export_items_on_created_by_user_id"
+    t.index ["material_id", "profile"], name: "index_material_export_items_on_material_id_and_profile", unique: true
+    t.index ["material_id"], name: "index_material_export_items_on_material_id"
+    t.index ["profile", "export_path"], name: "index_material_export_items_on_profile_and_export_path", unique: true
+  end
+
+  create_table "material_import_blocks", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
+    t.string "match_kind", null: false
+    t.string "sha256"
+    t.string "external_path_pattern"
+    t.string "reason", null: false
+    t.text "note"
+    t.bigint "created_by_user_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["created_by_user_id"], name: "index_material_import_blocks_on_created_by_user_id"
+  end
+
+  create_table "material_sync_sources", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
+    t.string "name", null: false
+    t.string "source_kind", null: false
+    t.string "source_uri"
+    t.string "source_path"
+    t.string "source_file_id"
+    t.string "profile", default: "legacy_drive", null: false
+    t.boolean "enabled", default: true, null: false
+    t.string "default_tag_name"
+    t.string "export_path_prefix"
+    t.datetime "last_synced_at"
+    t.bigint "created_by_user_id"
+    t.bigint "updated_by_user_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["created_by_user_id"], name: "index_material_sync_sources_on_created_by_user_id"
+    t.index ["enabled"], name: "index_material_sync_sources_on_enabled"
+    t.index ["source_kind"], name: "index_material_sync_sources_on_source_kind"
+    t.index ["updated_by_user_id"], name: "index_material_sync_sources_on_updated_by_user_id"
+  end
+
+  create_table "material_sync_suppressions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
+    t.string "source_kind", null: false
+    t.string "source_uri"
+    t.string "drive_path"
+    t.string "drive_file_id"
+    t.string "normalized_source_key", null: false
+    t.string "reason", null: false
+    t.bigint "created_by_user_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["created_by_user_id"], name: "index_material_sync_suppressions_on_created_by_user_id"
+    t.index ["drive_file_id"], name: "index_material_sync_suppressions_on_drive_file_id"
+    t.index ["normalized_source_key"], name: "index_material_sync_suppressions_on_normalized_source_key", unique: true
+    t.index ["source_kind"], name: "index_material_sync_suppressions_on_source_kind"
+  end
+
   create_table "material_versions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.bigint "material_id", null: false
     t.integer "version_no", null: false
@@ -141,14 +204,31 @@ ActiveRecord::Schema[8.0].define(version: 2026_06_22_020000) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "discarded_at"
+    t.string "event_type", null: false
+    t.string "tag_name"
+    t.string "tag_category"
+    t.json "export_paths_json"
+    t.bigint "file_blob_id"
+    t.string "file_filename"
+    t.string "file_content_type"
+    t.bigint "file_byte_size"
+    t.string "file_checksum"
+    t.string "file_sha256"
+    t.string "source_kind"
+    t.string "source_uri"
+    t.string "source_path"
+    t.string "source_file_id"
+    t.string "normalized_source_key"
     t.index ["created_by_user_id"], name: "index_material_versions_on_created_by_user_id"
     t.index ["discarded_at"], name: "index_material_versions_on_discarded_at"
+    t.index ["file_blob_id"], name: "index_material_versions_on_file_blob_id"
     t.index ["material_id", "version_no"], name: "index_material_versions_on_material_id_and_version_no", unique: true
     t.index ["material_id"], name: "index_material_versions_on_material_id"
     t.index ["parent_id"], name: "index_material_versions_on_parent_id"
     t.index ["tag_id"], name: "index_material_versions_on_tag_id"
     t.index ["updated_by_user_id"], name: "index_material_versions_on_updated_by_user_id"
     t.index ["url"], name: "index_material_versions_on_url"
+    t.check_constraint "`event_type` in (_utf8mb4'create',_utf8mb4'update',_utf8mb4'discard',_utf8mb4'restore')", name: "material_versions_event_type_valid"
   end
 
   create_table "materials", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
@@ -161,6 +241,14 @@ ActiveRecord::Schema[8.0].define(version: 2026_06_22_020000) do
     t.datetime "updated_at", null: false
     t.datetime "discarded_at"
     t.virtual "active_url", type: :string, as: "if((`discarded_at` is null),`url`,NULL)"
+    t.integer "version_no", default: 1, null: false
+    t.string "source_kind"
+    t.string "source_uri"
+    t.string "source_path"
+    t.string "source_file_id"
+    t.string "normalized_source_key"
+    t.virtual "active_normalized_source_key", type: :string, as: "if((`discarded_at` is null),`normalized_source_key`,NULL)"
+    t.index ["active_normalized_source_key"], name: "index_materials_on_active_normalized_source_key", unique: true
     t.index ["active_url"], name: "index_materials_on_active_url", unique: true
     t.index ["created_by_user_id"], name: "index_materials_on_created_by_user_id"
     t.index ["discarded_at"], name: "index_materials_on_discarded_at"
@@ -586,6 +674,12 @@ ActiveRecord::Schema[8.0].define(version: 2026_06_22_020000) do
   add_foreign_key "gekanator_question_suggestions", "users"
   add_foreign_key "gekanator_questions", "gekanator_question_suggestions"
   add_foreign_key "gekanator_questions", "users", column: "created_by_id"
+  add_foreign_key "material_export_items", "materials"
+  add_foreign_key "material_export_items", "users", column: "created_by_user_id"
+  add_foreign_key "material_import_blocks", "users", column: "created_by_user_id"
+  add_foreign_key "material_sync_sources", "users", column: "created_by_user_id"
+  add_foreign_key "material_sync_sources", "users", column: "updated_by_user_id"
+  add_foreign_key "material_sync_suppressions", "users", column: "created_by_user_id"
   add_foreign_key "material_versions", "materials"
   add_foreign_key "material_versions", "materials", column: "parent_id"
   add_foreign_key "material_versions", "tags"

backend/db/seeds.rb

@@ -7,3 +7,21 @@
 #   ["Action", "Comedy", "Drama", "Horror"].each do |genre_name|
 #     MovieGenre.find_or_create_by!(name: genre_name)
 #   end
+
+material_sync_source_uri = ENV['MATERIAL_SYNC_SOURCE_URI']
+material_sync_source_file_id = ENV['MATERIAL_SYNC_SOURCE_FILE_ID']
+
+if material_sync_source_uri.present? || material_sync_source_file_id.present?
+  source = MaterialSyncSource.find_or_initialize_by(
+      name: ENV.fetch('MATERIAL_SYNC_SOURCE_NAME', 'Legacy URI material sync'))
+  source.assign_attributes(
+      source_kind: ENV.fetch('MATERIAL_SYNC_SOURCE_KIND', 'uri'),
+      source_uri: material_sync_source_uri,
+      source_path: ENV['MATERIAL_SYNC_SOURCE_PATH'],
+      source_file_id: material_sync_source_file_id,
+      profile: ENV.fetch('MATERIAL_SYNC_SOURCE_PROFILE', 'legacy_drive'),
+      enabled: ENV.fetch('MATERIAL_SYNC_SOURCE_ENABLED', 'true') != 'false',
+      default_tag_name: ENV['MATERIAL_SYNC_SOURCE_TAG_NAME'],
+      export_path_prefix: ENV['MATERIAL_SYNC_EXPORT_PATH_PREFIX'])
+  source.save!
+end