#306

backend/app/services/material_sync_runner.rb

@@ -0,0 +1,212 @@
+require 'digest'
+
+class MaterialSyncRunner
+  Result = Struct.new(:imported, :updated, :suppressed, :failed, :errors, keyword_init: true)
+
+  def self.sync_enabled!
+    results = MaterialSyncSource.enabled.order(:id).map { |source| new(source).sync! }
+
+    Result.new(imported: results.sum(&:imported),
+               updated: results.sum(&:updated),
+               suppressed: results.sum(&:suppressed),
+               failed: results.sum(&:failed),
+               errors: results.flat_map(&:errors))
+  end
+
+  def initialize source
+    @source = source
+  end
+
+  def sync!
+    result = Result.new(imported: 0, updated: 0, suppressed: 0, failed: 0, errors: [])
+
+    candidates.each do |candidate|
+      sync_candidate!(candidate, result)
+    end
+
+    @source.update!(last_synced_at: Time.current)
+    log_result(result)
+    result
+  rescue NotImplementedError, StandardError => e
+    result.failed += 1
+    result.errors << { source_id: @source.id, error: e.message }
+    log_result(result)
+    result
+  end
+
+  private
+
+  def candidates
+    case @source.source_kind
+    when 'uri'
+      [uri_candidate]
+    when 'google_drive_path'
+      google_drive_path_candidates
+    when 'google_drive_file'
+      [google_drive_file_candidate]
+    when 'legacy_drive_path'
+      raise NotImplementedError, 'legacy_drive_path material sync is not implemented'
+    else
+      raise NotImplementedError, "Unsupported material sync source_kind: #{ @source.source_kind }"
+    end
+  end
+
+  def uri_candidate
+    { source_kind: @source.source_kind,
+      source_uri: @source.source_uri,
+      source_path: @source.source_path,
+      source_file_id: @source.source_file_id,
+      url: @source.source_uri,
+      tag: nil,
+      created_by_user: @source.created_by_user,
+      updated_by_user: @source.updated_by_user || @source.created_by_user }
+  end
+
+  def sync_candidate! candidate, result
+    block = MaterialImportBlockMatcher.match_for_sha256(candidate[:file_sha256])
+    if block
+      result.suppressed += 1
+      Rails.logger.info(
+          material_sync_log(action: 'suppressed',
+                            reason: block.reason,
+                            normalized_source_key: candidate_normalized_source_key(candidate)))
+      return
+    end
+
+    suppression = suppression_for(candidate)
+
+    if suppression
+      result.suppressed += 1
+      Rails.logger.info(
+          material_sync_log(action: 'suppressed',
+                            normalized_source_key: suppression.normalized_source_key))
+      return
+    end
+
+    import = MaterialSyncImporter.import!(candidate)
+    result.public_send("#{ import.action }=", result.public_send(import.action) + 1)
+  rescue StandardError => e
+    result.failed += 1
+    result.errors << { source_id: @source.id,
+                       normalized_source_key: candidate_normalized_source_key(candidate),
+                       error: e.message }
+    Rails.logger.warn(
+        material_sync_log(action: 'failed',
+                          error: e.message,
+                          normalized_source_key: candidate_normalized_source_key(candidate)))
+  ensure
+    close_candidate_tempfile(candidate)
+  end
+
+  def google_drive_path_candidates
+    folder_id = google_drive_folder_id
+    Enumerator.new do |entries|
+      drive_client.list_material_files_under_folder(folder_id).each do |entry|
+        entries << build_google_drive_candidate(entry)
+      end
+    end
+  end
+
+  def google_drive_file_candidate
+    build_google_drive_candidate(drive_client.fetch_material_file(google_drive_file_id),
+                                 single_file: true)
+  end
+
+  def build_google_drive_candidate entry, single_file: false
+    relative_path =
+        if single_file
+          nil
+        else
+          entry[:relative_path]
+        end
+    export_path =
+        MaterialSyncExportPath.build(prefix: @source.export_path_prefix,
+                                     relative_path:,
+                                     filename: entry[:name],
+                                     source_file_id: entry[:id])
+    tempfile =
+        drive_client.download_to_tempfile(entry[:id], filename: entry[:name])
+    file_sha256 = Digest::SHA256.file(tempfile.path).hexdigest
+    tempfile.rewind
+
+    { source_kind: 'google_drive_file',
+      source_uri: entry[:web_view_link] || google_drive_file_url(entry[:id]),
+      source_path: relative_path.presence || MaterialSyncExportPath.normalize_path(entry[:name]),
+      source_file_id: entry[:id],
+      filename: entry[:name],
+      content_type: entry[:mime_type],
+      file_tempfile: tempfile,
+      file_sha256:,
+      export_path:,
+      profile: @source.profile,
+      tag: nil,
+      url: nil,
+      created_by_user: @source.created_by_user,
+      updated_by_user: @source.updated_by_user || @source.created_by_user }
+  end
+
+  def candidate_normalized_source_key candidate
+    MaterialSyncSuppression.normalize_source_key(source_kind: candidate[:source_kind],
+                                                 source_uri: candidate[:source_uri],
+                                                 source_path: candidate[:source_path],
+                                                 source_file_id: candidate[:source_file_id])
+  end
+
+  def suppression_for candidate
+    if candidate[:source_kind] == 'google_drive_file' && candidate[:source_path].present?
+      return MaterialSyncSuppressionMatcher.match_google_drive_candidate(
+          drive_file_id: candidate[:source_file_id],
+          relative_path: candidate[:source_path])
+    end
+
+    MaterialSyncSuppressionMatcher.match(source_kind: candidate[:source_kind],
+                                         source_uri: candidate[:source_uri],
+                                         source_path: candidate[:source_path],
+                                         source_file_id: candidate[:source_file_id])
+  end
+
+  def drive_client
+    @drive_client ||= GoogleDrive::ApiClient.new
+  end
+
+  def google_drive_folder_id
+    google_drive_target_id.tap do |id|
+      raise ArgumentError, 'google_drive_path source_file_id or source_uri is required' if id.blank?
+    end
+  end
+
+  def google_drive_file_id
+    google_drive_target_id.tap do |id|
+      raise ArgumentError, 'google_drive_file source_file_id or source_uri is required' if id.blank?
+    end
+  end
+
+  def google_drive_target_id
+    @source.source_file_id.presence || drive_client.extract_file_id(@source.source_uri)
+  end
+
+  def google_drive_file_url file_id
+    "https://drive.google.com/file/d/#{ file_id }/view"
+  end
+
+  def close_candidate_tempfile candidate
+    tempfile = candidate[:file_tempfile]
+    return unless tempfile
+
+    tempfile.close! unless tempfile.closed?
+  rescue StandardError
+    nil
+  end
+
+  def log_result result
+    Rails.logger.info(material_sync_log(imported: result.imported,
+                                        updated: result.updated,
+                                        suppressed: result.suppressed,
+                                        failed: result.failed))
+  end
+
+  def material_sync_log fields
+    { material_sync_source_id: @source.id,
+      material_sync_source_name: @source.name }.merge(fields).to_json
+  end
+end