namespace :nico do desc 'ニコニコ DB 同期' task sync: :environment do require 'json' require 'nokogiri' require 'open-uri' require 'open3' require 'set' require 'time' fetch_thumbnail = -> url do html = URI.open(url, read_timeout: 60, 'User-Agent' => 'Mozilla/5.0').read doc = Nokogiri::HTML(html) doc.at('meta[name="thumbnail"]')&.[]('content').presence end def sync_post_tags! post, desired_tag_ids, current_tag_ids: nil current_tag_ids ||= PostTag.where(post_id: post.id).pluck(:tag_id).to_set desired_tag_ids = desired_tag_ids.compact.to_set to_add = desired_tag_ids - current_tag_ids to_remove = current_tag_ids - desired_tag_ids Tag.where(id: to_add.to_a).find_each do |tag| begin PostTag.create!(post:, tag:) rescue ActiveRecord::RecordNotUnique ; end end PostTag.where(post_id: post.id, tag_id: to_remove.to_a).find_each(&:destroy!) end def sync_post_external_tags! post, desired_external_tag_ids, current_external_tag_ids: nil current_external_tag_ids ||= PostExternalTag.where(post_id: post.id).pluck(:external_tag_id).to_set desired_external_tag_ids = desired_external_tag_ids.compact.to_set to_add = desired_external_tag_ids - current_external_tag_ids to_remove = current_external_tag_ids - desired_external_tag_ids ExternalTag.where(id: to_add.to_a).find_each do |external_tag| begin PostExternalTag.create!(post:, external_tag:) rescue ActiveRecord::RecordNotUnique ; end end PostExternalTag .where(post_id: post.id, external_tag_id: to_remove.to_a) .find_each(&:destroy!) end mysql_user = ENV['MYSQL_USER'] mysql_pass = ENV['MYSQL_PASS'] nizika_nico_path = ENV['NIZIKA_NICO_PATH'] stdout, stderr, status = Open3.capture3( { 'MYSQL_USER' => mysql_user, 'MYSQL_PASS' => mysql_pass }, 'python3', "#{ nizika_nico_path }/get_videos.py") unless status.success? warn stderr abort end data = JSON.parse(stdout) data.each do |datum| code = datum['code'] post = Post .where('url REGEXP ?', "nicovideo\\.jp/watch/#{ Regexp.escape(code) }([^0-9]|$)") .first title = datum['title'] original_created_at = datum['uploaded_at'] && Time.strptime(datum['uploaded_at'], '%Y-%m-%d %H:%M:%S') original_created_from = original_created_at&.change(sec: 0) original_created_before = original_created_from&.+(1.minute) post_created = false post_changed = false if post attrs = { title:, original_created_from:, original_created_before: } unless post.thumbnail.attached? thumbnail_base = fetch_thumbnail.(post.url) rescue nil if thumbnail_base.present? attrs[:thumbnail_base] = thumbnail_base begin post.attach_thumbnail_from_url!(thumbnail_base) rescue Post::RemoteThumbnailFetchFailed => e payload = { post_id: post.id, thumbnail_base:, error: e.class.name, message: e.message } Rails.logger.info("nico_sync_thumbnail_fetch_failed #{ payload.to_json }") end end end post.assign_attributes(attrs) post_changed = post.changed? if post_changed post.save! end else post_created = true url = "https://www.nicovideo.jp/watch/#{ code }" thumbnail_base = fetch_thumbnail.(url) rescue nil post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil, original_created_from:, original_created_before:) if thumbnail_base.present? begin post.attach_thumbnail_from_url!(thumbnail_base) rescue Post::RemoteThumbnailFetchFailed => e payload = { post_id: nil, thumbnail_base:, error: e.class.name, message: e.message } Rails.logger.info("nico_sync_thumbnail_fetch_failed #{ payload.to_json }") end end post.save! sync_post_tags!(post, [Tag.tagme.id, Tag.bot.id, Tag.niconico.id, Tag.video.id]) end # 既存のタグ Id. 集合 kept_tag_ids = post.tags.pluck(:id).to_set # 既存の外部タグ Id. 集合 kept_external_tag_ids = post.external_tags.nico.pluck(:id).to_set # 記載すべき外部タグ Id. のリスト desired_external_tag_ids = [] # 記載すべき内部タグ Id. のリスト desired_tag_ids = kept_tag_ids.to_a datum['tags'].each do |raw| name = TagNameSanitisationRule.sanitise(raw) tag = ExternalTag.find_or_create_by!(platform: :nico, name:) unless tag.nico_tag_versions.exists? NicoTagVersionRecorder.record!(external_tag: tag, event_type: :create, created_by_user: nil) end desired_external_tag_ids << tag.id # 新たに記載される外部タグと連携される内部タグを記載 # 連携タグは記載すれども消除せず. unless tag.id.in?(kept_external_tag_ids) desired_tag_ids.concat(tag.linked_tags.pluck(:id)) end end deerjikist = Deerjikist.find_by(platform: :nico, code: datum['user']) if deerjikist desired_tag_ids << deerjikist.tag_id elsif !(Tag.where(id: kept_tag_ids).where(category: :deerjikist).exists?) desired_tag_ids << Tag.no_deerjikist.id end desired_external_tag_ids.uniq! desired_tag_ids.uniq! # 外部タグの記載に際しては “bot 操作” タグを記載しなぃ. if kept_tag_ids != desired_tag_ids.to_set desired_tag_ids << Tag.bot.id desired_tag_ids.uniq! end tags_changed = kept_tag_ids != desired_tag_ids.to_set || kept_external_tag_ids != desired_external_tag_ids.to_set sync_post_tags!(post, desired_tag_ids, current_tag_ids: kept_tag_ids) sync_post_external_tags!(post, desired_external_tag_ids, current_external_tag_ids: kept_external_tag_ids) if post_created PostVersionRecorder.record!(post:, event_type: :create, created_by_user: nil) elsif post_changed || tags_changed PostVersionRecorder.ensure_snapshot!(post, created_by_user: nil) PostVersionRecorder.record!(post:, event_type: :update, created_by_user: nil) end end end end