require 'open-uri' require 'set' require 'time' module Youtube class Sync def initialize client: ApiClient.new @client = client end def sync! video_ids = discover_video_ids return if video_ids.empty? video_ids.each_slice(50) do |ids| @client.videos(ids).fetch('items', []).each do |item| sync_video!(VideoItem.new(item)) end end end private def discover_video_ids ids = Set.new query_terms.each do |q| response = @client.search_videos(q:, published_after: sync_since) response.fetch('items', []).each do |item| video_id = item.dig('id', 'videoId') ids << video_id if video_id.present? end end playlist_ids.each do |playlist_id| response = @client.playlist_items(playlist_id:) response.fetch('items', []).each do |item| video_id = item.dig('contentDetails', 'videoId') ids << video_id if video_id.present? end end ids.to_a end def sync_video! video post = Post.where('url REGEXP ?', youtube_url_regexp(video.id)).first original_created_from = video.published_at.change(sec: 0) original_created_before = original_created_from + 1.minute post_created = false post_changed = false if post post.assign_attributes(title: video.title, original_created_from:, original_created_before:, thumbnail_base: video.thumbnail_url) post_changed = post.changed? post.save! if post_changed attach_thumbnail_if_needed!(post, video.thumbnail_url) else post_created = true post = Post.create!( title: video.title, url: video.url, thumbnail_base: video.thumbnail_url, uploaded_user: nil, original_created_from:, original_created_before:) attach_thumbnail_if_needed!(post, video.thumbnail_url) sync_post_tags!(post, [Tag.tagme.id, Tag.bot.id, Tag.youtube.id, Tag.video.id]) end kept_tag_ids = post.tags.pluck(:id).to_set desired_tag_ids = kept_tag_ids.to_a deerjikist = Deerjikist.find_by(platform: :youtube, code: video.channel_id) if deerjikist desired_tag_ids << deerjikist.tag_id elsif post.tags.where(category: :deerjikist).none? desired_tag_ids << Tag.no_deerjikist.id end desired_tag_ids.uniq! sync_post_tags!(post, desired_tag_ids, current_tag_ids: kept_tag_ids) if post_created PostVersionRecorder.record!(post:, event_type: :create, created_by_user: nil) elsif post_changed || kept_tag_ids != desired_tag_ids.to_set PostVersionRecorder.ensure_snapshot!(post, created_by_user: nil) PostVersionRecorder.record!(post:, event_type: :update, created_by_user: nil) end end def sync_post_tags! post, desired_tag_ids, current_tag_ids: nil current_tag_ids ||= PostTag.kept.where(post_id: post.id).pluck(:tag_id).to_set desired_tag_ids = desired_tag_ids.compact.to_set to_add = desired_tag_ids - current_tag_ids to_remove = current_tag_ids - desired_tag_ids Tag.where(id: to_add.to_a).find_each do |tag| begin PostTag.create!(post:, tag:) rescue ActiveRecord::RecordNotUnique ; end end PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each do |pt| pt.discard_by!(nil) end end def attach_thumbnail_if_needed! post, thumbnail_url return if post.thumbnail.attached? return if thumbnail_url.blank? post.thumbnail.attach( io: URI.open(thumbnail_url), filename: File.basename(URI.parse(thumbnail_url).path), content_type: 'image/jpeg') post.resized_thumbnail! end def youtube_url_regexp id escaped = Regexp.escape(id) "(youtube\\.com/watch\\?v=#{ escaped }|youtu\\.be/#{ escaped })([^A-Za-z0-9_-]|$)" end def query_terms = ['ぼざろクリーチャーシリーズ', '伊地知ニジカ', '伊地知虹鹿'] def playlist_ids ['PLrOch4zHkI5vu29b-f9umUQQ4tQkuWLPX', 'PLrOch4zHkI5vOK0RaytQq6PbucxQkkL0K', 'PLrOch4zHkI5tdwm9vSegiDQJOM-hgpcOC'] end def sync_since = 14.days.ago end end