196 行
6.6 KiB
Ruby
196 行
6.6 KiB
Ruby
namespace :nico do
|
|
desc 'ニコニコ DB 同期'
|
|
task sync: :environment do
|
|
require 'json'
|
|
require 'nokogiri'
|
|
require 'open-uri'
|
|
require 'open3'
|
|
require 'set'
|
|
require 'time'
|
|
|
|
fetch_thumbnail = -> url do
|
|
html = URI.open(url, read_timeout: 60, 'User-Agent' => 'Mozilla/5.0').read
|
|
doc = Nokogiri::HTML(html)
|
|
|
|
doc.at('meta[name="thumbnail"]')&.[]('content').presence
|
|
end
|
|
|
|
def sync_post_tags! post, desired_tag_ids, current_tag_ids: nil
|
|
current_tag_ids ||= PostTag.where(post_id: post.id).pluck(:tag_id).to_set
|
|
desired_tag_ids = desired_tag_ids.compact.to_set
|
|
|
|
to_add = desired_tag_ids - current_tag_ids
|
|
to_remove = current_tag_ids - desired_tag_ids
|
|
|
|
Tag.where(id: to_add.to_a).find_each do |tag|
|
|
begin
|
|
PostTag.create!(post:, tag:)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
;
|
|
end
|
|
end
|
|
|
|
PostTag.where(post_id: post.id, tag_id: to_remove.to_a).find_each(&:destroy!)
|
|
end
|
|
|
|
def sync_post_external_tags! post, desired_external_tag_ids, current_external_tag_ids: nil
|
|
current_external_tag_ids ||=
|
|
PostExternalTag.where(post_id: post.id).pluck(:external_tag_id).to_set
|
|
desired_external_tag_ids = desired_external_tag_ids.compact.to_set
|
|
|
|
to_add = desired_external_tag_ids - current_external_tag_ids
|
|
to_remove = current_external_tag_ids - desired_external_tag_ids
|
|
|
|
ExternalTag.where(id: to_add.to_a).find_each do |external_tag|
|
|
begin
|
|
PostExternalTag.create!(post:, external_tag:)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
;
|
|
end
|
|
end
|
|
|
|
PostExternalTag
|
|
.where(post_id: post.id, external_tag_id: to_remove.to_a)
|
|
.find_each(&:destroy!)
|
|
end
|
|
|
|
mysql_user = ENV['MYSQL_USER']
|
|
mysql_pass = ENV['MYSQL_PASS']
|
|
nizika_nico_path = ENV['NIZIKA_NICO_PATH']
|
|
stdout, stderr, status = Open3.capture3(
|
|
{ 'MYSQL_USER' => mysql_user, 'MYSQL_PASS' => mysql_pass },
|
|
'python3', "#{ nizika_nico_path }/get_videos.py")
|
|
|
|
unless status.success?
|
|
warn stderr
|
|
abort
|
|
end
|
|
|
|
data = JSON.parse(stdout)
|
|
data.each do |datum|
|
|
code = datum['code']
|
|
post =
|
|
Post
|
|
.where('url REGEXP ?', "nicovideo\\.jp/watch/#{ Regexp.escape(code) }([^0-9]|$)")
|
|
.first
|
|
|
|
title = datum['title']
|
|
original_created_at = datum['uploaded_at'] &&
|
|
Time.strptime(datum['uploaded_at'], '%Y-%m-%d %H:%M:%S')
|
|
original_created_from = original_created_at&.change(sec: 0)
|
|
original_created_before = original_created_from&.+(1.minute)
|
|
|
|
post_created = false
|
|
post_changed = false
|
|
|
|
if post
|
|
attrs = { title:, original_created_from:, original_created_before: }
|
|
|
|
unless post.thumbnail.attached?
|
|
thumbnail_base = fetch_thumbnail.(post.url) rescue nil
|
|
if thumbnail_base.present?
|
|
attrs[:thumbnail_base] = thumbnail_base
|
|
begin
|
|
post.attach_thumbnail_from_url!(thumbnail_base)
|
|
rescue Post::RemoteThumbnailFetchFailed => e
|
|
payload = {
|
|
post_id: post.id,
|
|
thumbnail_base:,
|
|
error: e.class.name,
|
|
message: e.message }
|
|
Rails.logger.info("nico_sync_thumbnail_fetch_failed #{ payload.to_json }")
|
|
end
|
|
end
|
|
end
|
|
|
|
post.assign_attributes(attrs)
|
|
post_changed = post.changed?
|
|
if post_changed
|
|
post.save!
|
|
end
|
|
else
|
|
post_created = true
|
|
url = "https://www.nicovideo.jp/watch/#{ code }"
|
|
thumbnail_base = fetch_thumbnail.(url) rescue nil
|
|
post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil,
|
|
original_created_from:, original_created_before:)
|
|
if thumbnail_base.present?
|
|
begin
|
|
post.attach_thumbnail_from_url!(thumbnail_base)
|
|
rescue Post::RemoteThumbnailFetchFailed => e
|
|
payload = {
|
|
post_id: nil,
|
|
thumbnail_base:,
|
|
error: e.class.name,
|
|
message: e.message }
|
|
Rails.logger.info("nico_sync_thumbnail_fetch_failed #{ payload.to_json }")
|
|
end
|
|
end
|
|
post.save!
|
|
sync_post_tags!(post, [Tag.tagme.id, Tag.bot.id, Tag.niconico.id, Tag.video.id])
|
|
end
|
|
|
|
# 既存のタグ Id. 集合
|
|
kept_tag_ids = post.tags.pluck(:id).to_set
|
|
|
|
# 既存の外部タグ Id. 集合
|
|
kept_external_tag_ids = post.external_tags.pluck(:id).to_set
|
|
|
|
# 記載すべき外部タグ Id. のリスト
|
|
desired_external_tag_ids = []
|
|
|
|
# 記載すべき内部タグ Id. のリスト
|
|
desired_tag_ids = kept_tag_ids.to_a
|
|
|
|
datum['tags'].each do |raw|
|
|
tag = ExternalTag.find_or_create_by!(platform: :nico, name: raw)
|
|
|
|
unless tag.nico_tag_versions.exists?
|
|
NicoTagVersionRecorder.record!(external_tag: tag,
|
|
event_type: :create,
|
|
created_by_user: nil)
|
|
end
|
|
|
|
desired_external_tag_ids << tag.id
|
|
|
|
# 新たに記載される外部タグと連携される内部タグを記載
|
|
# 連携タグは記載すれども消除せず．
|
|
unless tag.id.in?(kept_external_tag_ids)
|
|
desired_tag_ids.concat(tag.linked_tags.pluck(:id))
|
|
end
|
|
end
|
|
|
|
deerjikist = Deerjikist.find_by(platform: :nico, code: datum['user'])
|
|
if deerjikist
|
|
desired_tag_ids << deerjikist.tag_id
|
|
elsif !(Tag.where(id: kept_tag_ids).where(category: :deerjikist).exists?)
|
|
desired_tag_ids << Tag.no_deerjikist.id
|
|
end
|
|
|
|
desired_external_tag_ids.uniq!
|
|
desired_tag_ids.uniq!
|
|
|
|
# 外部タグの記載に際しては “bot 操作” タグを記載しなぃ．
|
|
if kept_tag_ids != desired_tag_ids.to_set
|
|
desired_tag_ids << Tag.bot.id
|
|
desired_tag_ids.uniq!
|
|
end
|
|
|
|
tags_changed =
|
|
kept_tag_ids != desired_tag_ids.to_set ||
|
|
kept_external_tag_ids != desired_external_tag_ids.to_set
|
|
|
|
sync_post_tags!(post, desired_tag_ids, current_tag_ids: kept_tag_ids)
|
|
sync_post_external_tags!(post, desired_external_tag_ids,
|
|
current_external_tag_ids: kept_external_tag_ids)
|
|
|
|
if post_created
|
|
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: nil)
|
|
elsif post_changed || tags_changed
|
|
PostVersionRecorder.ensure_snapshot!(post, created_by_user: nil)
|
|
PostVersionRecorder.record!(post:, event_type: :update, created_by_user: nil)
|
|
end
|
|
end
|
|
end
|
|
end
|