miteruzo
/
btrc-hub
1
0
Fork 0
Code Issues 68 Pull Requests 1 Releases 0 Wiki Activity
Browse Source

YouTube の自動同期 (#314) (#340) 

#314

#314

#314

#314

#314

Co-authored-by: miteruzo <miteruzo@naver.com>
Reviewed-on: https://git.miteruzo.com/miteruzo/btrc-hub/pulls/340
main
みてるぞ 4 days ago
parent
fcd3b87b2a
commit
5002859fc8
10 changed files with 846 additions and 0 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -0
      backend/app/models/tag.rb
  2. +73
    -0
      backend/app/services/youtube/api_client.rb
  3. +168
    -0
      backend/app/services/youtube/sync.rb
  4. +32
    -0
      backend/app/services/youtube/video_item.rb
  5. +8
    -0
      backend/config/schedule.rb
  6. +6
    -0
      backend/lib/tasks/sync_posts.rake
  7. +130
    -0
      backend/spec/services/youtube/api_client_spec.rb
  8. +310
    -0
      backend/spec/services/youtube/sync_spec.rb
  9. +93
    -0
      backend/spec/services/youtube/video_item_spec.rb
  10. +25
    -0
      backend/spec/tasks/post_sync_spec.rb

+ 1
- 0
backend/app/models/tag.rb View File

@@ -84,6 +84,7 @@ class Tag < ApplicationRecord
def self.no_deerjikist = find_or_create_by_tag_name!('ニジラー情報不詳', category: :meta)
def self.video = find_or_create_by_tag_name!('動画', category: :meta)
def self.niconico = find_or_create_by_tag_name!('ニコニコ', category: :meta)
def self.youtube = find_or_create_by_tag_name!('YouTube', category: :meta)

def self.normalise_tags tag_names, with_tagme: true,
with_no_deerjikist: true,


+ 73
- 0
backend/app/services/youtube/api_client.rb View File

@@ -0,0 +1,73 @@
require 'json'
require 'net/http'
require 'uri'


module Youtube
class ApiClient
ENDPOINT = 'https://www.googleapis.com/youtube/v3'

def initialize api_key: ENV.fetch('YOUTUBE_API_KEY')
@api_key = api_key
end

def search_videos q:, published_after: nil, published_before: nil, page_token: nil
get_json('/search', {
part: 'snippet',
type: 'video',
q:,
order: 'date',
maxResults: 50,
regionCode: 'JP',
relevanceLanguage: 'ja',
publishedAfter: published_after&.iso8601,
publishedBefore: published_before&.iso8601,
pageToken: page_token }.compact)
end

def videos ids
return { 'items' => [] } if ids.empty?

get_json('/videos', part: 'snippet,status,contentDetails', id: ids.join(','))
end

def playlist_items playlist_id:, page_token: nil
get_json('/playlistItems', {
part: 'snippet,contentDetails,status',
playlistId: playlist_id,
maxResults: 50,
pageToken: page_token }.compact)
end

def channel id: nil, handle: nil
raise ArgumentError, 'id or handle is required' if id.present? == handle.present?

params = { part: 'snippet,contentDetails' }
params[:id] = id if id.present?
params[:forHandle] = handle if handle.present?

get_json('/channels', params)
end

private

def get_json path, params
uri = URI(ENDPOINT + path)
uri.query = URI.encode_www_form(params.merge(key: @api_key))

response = Net::HTTP.start(uri.host,
uri.port,
use_ssl: true,
open_timeout: 10,
read_timeout: 30) do |http|
http.get(uri)
end

unless response.is_a?(Net::HTTPSuccess)
raise "YouTube API error: #{ response.code } #{ response.body }"
end

JSON.parse(response.body)
end
end
end

+ 168
- 0
backend/app/services/youtube/sync.rb View File

@@ -0,0 +1,168 @@
require 'open-uri'
require 'set'
require 'time'


module Youtube
class Sync
def initialize client: ApiClient.new
@client = client
end

def sync!
video_ids = discover_video_ids
return if video_ids.empty?

video_ids.each_slice(50) do |ids|
@client.videos(ids).fetch('items', []).each do |item|
sync_video!(VideoItem.new(item))
end
end
end

private

def discover_video_ids
ids = Set.new

query_terms.each do |q|
response = @client.search_videos(q:, published_after: sync_since)

response.fetch('items', []).each do |item|
video_id = item.dig('id', 'videoId')
ids << video_id if video_id.present?
end
end

playlist_ids.each do |playlist_id|
each_playlist_item(playlist_id) do |item|
video_id = item.dig('contentDetails', 'videoId')
video_id ||= item.dig('snippet', 'resourceId', 'videoId')

ids << video_id if video_id.present?
end
end

ids.to_a
end

def sync_video! video
post = Post.where('url REGEXP ?', youtube_url_regexp(video.id)).first

original_created_from = video.published_at.change(sec: 0)
original_created_before = original_created_from + 1.minute

post_created = false
post_changed = false

if post
post.assign_attributes(title: video.title,
original_created_from:,
original_created_before:,
thumbnail_base: video.thumbnail_url)

post_changed = post.changed?
post.save! if post_changed

attach_thumbnail_if_needed!(post, video.thumbnail_url)
else
post_created = true
post = Post.create!(
title: video.title,
url: video.url,
thumbnail_base: video.thumbnail_url,
uploaded_user_id: nil,
original_created_from:,
original_created_before:)

attach_thumbnail_if_needed!(post, video.thumbnail_url)

sync_post_tags!(post, [Tag.tagme.id, Tag.bot.id, Tag.youtube.id, Tag.video.id])
end

kept_tag_ids = post.tags.pluck(:id).to_set
desired_tag_ids = kept_tag_ids.to_a

deerjikist = Deerjikist.find_by(platform: :youtube, code: video.channel_id)
if deerjikist
desired_tag_ids.delete(Tag.no_deerjikist.id)
desired_tag_ids << deerjikist.tag_id
elsif post.tags.where(category: :deerjikist).none?
desired_tag_ids << Tag.no_deerjikist.id
end

desired_tag_ids.uniq!

sync_post_tags!(post, desired_tag_ids, current_tag_ids: kept_tag_ids)

if post_created
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: nil)
elsif post_changed || kept_tag_ids != desired_tag_ids.to_set
PostVersionRecorder.ensure_snapshot!(post, created_by_user: nil)
PostVersionRecorder.record!(post:, event_type: :update, created_by_user: nil)
end
end

def sync_post_tags! post, desired_tag_ids, current_tag_ids: nil
current_tag_ids ||= PostTag.kept.where(post_id: post.id).pluck(:tag_id).to_set
desired_tag_ids = desired_tag_ids.compact.to_set

to_add = desired_tag_ids - current_tag_ids
to_remove = current_tag_ids - desired_tag_ids

Tag.where(id: to_add.to_a).find_each do |tag|
begin
PostTag.create!(post:, tag:)
rescue ActiveRecord::RecordNotUnique
;
end
end

PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each do |pt|
pt.discard_by!(nil)
end
end

def attach_thumbnail_if_needed! post, thumbnail_url
return if post.thumbnail.attached?
return if thumbnail_url.blank?

post.thumbnail.attach(
io: URI.open(thumbnail_url),
filename: File.basename(URI.parse(thumbnail_url).path),
content_type: 'image/jpeg')

post.resized_thumbnail!
end

def youtube_url_regexp id
escaped = Regexp.escape(id)
"(youtube\\.com/watch\\?v=#{ escaped }|youtu\\.be/#{ escaped })([^A-Za-z0-9_-]|$)"
end

def query_terms = ['ぼざろクリーチャーシリーズ', '伊地知ニジカ', '伊地知虹鹿']

def playlist_ids
['PLrOch4zHkI5vu29b-f9umUQQ4tQkuWLPX',
'PLrOch4zHkI5vOK0RaytQq6PbucxQkkL0K',
'PLrOch4zHkI5tdwm9vSegiDQJOM-hgpcOC']
end

def sync_since = 14.days.ago

def each_playlist_item playlist_id
page_token = nil

loop do
response = @client.playlist_items(playlist_id:, page_token:)

response.fetch('items', []).each do |item|
yield item
end

page_token = response['nextPageToken']
break if page_token.blank?
end
end
end
end

+ 32
- 0
backend/app/services/youtube/video_item.rb View File

@@ -0,0 +1,32 @@
require 'time'


module Youtube
class VideoItem
attr_reader :id, :title, :channel_id, :published_at, :thumbnail_url, :raw_tags

def initialize item
snippet = item.fetch('snippet')

@id = item.fetch('id')
@title = snippet['title']
@channel_id = snippet['channelId']
@published_at = Time.iso8601(snippet['publishedAt'])
@thumbnail_url = pick_thumbnail(snippet['thumbnails'] || { })
@raw_tags = snippet['tags'] || []
end

def url = "https://www.youtube.com/watch?v=#{ @id }"

private

def pick_thumbnail thumbnails
['maxres', 'standard', 'high', 'medium', 'default'].each do |key|
url = thumbnails.dig(key, 'url')
return url if url.present?
end

nil
end
end
end

+ 8
- 0
backend/config/schedule.rb View File

@@ -17,3 +17,11 @@ every 1.day, at: '0:00 am' do
rake 'post_similarity:calc', environment: 'production'
rake 'tag_similarity:calc', environment: 'production'
end

every 1.day, at: '7:50 am' do
rake 'nico:export', environment: 'production'
end

every :hour do
rake 'post:sync', environment: 'production'
end

+ 6
- 0
backend/lib/tasks/sync_posts.rake View File

@@ -0,0 +1,6 @@
namespace :post do
desc '投稿同期（ニコニコ以外）'
task sync: :environment do
Youtube::Sync.new.sync!
end
end

+ 130
- 0
backend/spec/services/youtube/api_client_spec.rb View File

@@ -0,0 +1,130 @@
require 'rails_helper'

RSpec.describe Youtube::ApiClient do
let(:api_key) { 'test-api-key' }
let(:client) { described_class.new(api_key:) }

describe '#search_videos' do
it 'calls YouTube search API with expected params' do
published_after = Time.zone.parse('2026-05-01 00:00:00')
published_before = Time.zone.parse('2026-05-02 00:00:00')

expect(client).to receive(:get_json).with(
'/search',
{
part: 'snippet',
type: 'video',
q: 'ぼざろクリーチャー',
order: 'date',
maxResults: 50,
regionCode: 'JP',
relevanceLanguage: 'ja',
publishedAfter: published_after.iso8601,
publishedBefore: published_before.iso8601,
pageToken: 'NEXT'
}
).and_return({ 'items' => [] })

client.search_videos(
q: 'ぼざろクリーチャー',
published_after:,
published_before:,
page_token: 'NEXT'
)
end

it 'omits nil optional params' do
expect(client).to receive(:get_json).with(
'/search',
hash_excluding(:publishedAfter, :publishedBefore, :pageToken)
).and_return({ 'items' => [] })

client.search_videos(q: 'ぼざろクリーチャー')
end
end

describe '#videos' do
it 'returns empty items when ids are empty' do
expect(client).not_to receive(:get_json)

expect(client.videos([])).to eq({ 'items' => [] })
end

it 'calls videos API with comma separated ids' do
expect(client).to receive(:get_json).with(
'/videos',
{
part: 'snippet,status,contentDetails',
id: 'video-1,video-2'
}
).and_return({ 'items' => [] })

client.videos(['video-1', 'video-2'])
end
end

describe '#playlist_items' do
it 'calls playlistItems API with page token' do
expect(client).to receive(:get_json).with(
'/playlistItems',
{
part: 'snippet,contentDetails,status',
playlistId: 'PL123',
maxResults: 50,
pageToken: 'NEXT'
}
).and_return({ 'items' => [] })

client.playlist_items(playlist_id: 'PL123', page_token: 'NEXT')
end

it 'omits page token when nil' do
expect(client).to receive(:get_json).with(
'/playlistItems',
{
part: 'snippet,contentDetails,status',
playlistId: 'PL123',
maxResults: 50
}
).and_return({ 'items' => [] })

client.playlist_items(playlist_id: 'PL123')
end
end

describe '#channel' do
it 'calls channels API by id' do
expect(client).to receive(:get_json).with(
'/channels',
{
part: 'snippet,contentDetails',
id: 'UC123'
}
).and_return({ 'items' => [] })

client.channel(id: 'UC123')
end

it 'calls channels API by handle' do
expect(client).to receive(:get_json).with(
'/channels',
{
part: 'snippet,contentDetails',
forHandle: '@some_handle'
}
).and_return({ 'items' => [] })

client.channel(handle: '@some_handle')
end

it 'raises when neither id nor handle is given' do
expect { client.channel }.to raise_error(ArgumentError, 'id or handle is required')
end

it 'raises when both id and handle are given' do
expect do
client.channel(id: 'UC123', handle: '@some_handle')
end.to raise_error(ArgumentError, 'id or handle is required')
end
end
end

+ 310
- 0
backend/spec/services/youtube/sync_spec.rb View File

@@ -0,0 +1,310 @@
require 'rails_helper'

RSpec.describe Youtube::Sync do
let(:client) { instance_double(Youtube::ApiClient) }
let(:sync) { described_class.new(client:) }

before do
allow(PostVersionRecorder).to receive(:record!)
allow(PostVersionRecorder).to receive(:ensure_snapshot!)
allow(sync).to receive(:attach_thumbnail_if_needed!)
end

describe '#sync!' do
it 'returns without fetching video details when no video ids are discovered' do
allow(sync).to receive(:query_terms).and_return([])
allow(sync).to receive(:playlist_ids).and_return([])

expect(client).not_to receive(:videos)

sync.sync!
end

it 'discovers ids from search and all playlist pages' do
allow(sync).to receive(:query_terms).and_return(['ぼざろクリーチャー'])
allow(sync).to receive(:playlist_ids).and_return(['PL123'])
allow(sync).to receive(:sync_since).and_return(Time.zone.parse('2026-05-01 00:00:00'))

allow(client).to receive(:search_videos).with(
q: 'ぼざろクリーチャー',
published_after: Time.zone.parse('2026-05-01 00:00:00')
).and_return({
'items' => [
{
'id' => {
'videoId' => 'search-video-1'
}
}
]
})

allow(client).to receive(:playlist_items).with(
playlist_id: 'PL123',
page_token: nil
).and_return({
'items' => [
{
'contentDetails' => {
'videoId' => 'playlist-video-1'
}
}
],
'nextPageToken' => 'NEXT'
})

allow(client).to receive(:playlist_items).with(
playlist_id: 'PL123',
page_token: 'NEXT'
).and_return({
'items' => [
{
'snippet' => {
'resourceId' => {
'videoId' => 'playlist-video-2'
}
}
}
]
})

expect(client).to receive(:videos).with(
satisfy do |ids|
ids.sort == ['playlist-video-1', 'playlist-video-2', 'search-video-1']
end
).and_return({ 'items' => [] })

sync.sync!
end

it 'creates a YouTube post with default tags and no_deerjikist when no deerjikist mapping exists' do
Tag.tagme
Tag.bot
Tag.youtube
Tag.video
Tag.no_deerjikist

allow(sync).to receive(:query_terms).and_return([])
allow(sync).to receive(:playlist_ids).and_return(['PL123'])

allow(client).to receive(:playlist_items).with(
playlist_id: 'PL123',
page_token: nil
).and_return({
'items' => [
{
'contentDetails' => {
'videoId' => 'video-1'
}
}
]
})

allow(client).to receive(:videos).with(['video-1']).and_return({
'items' => [
youtube_video_item(
id: 'video-1',
title: 'YouTube テスト動画',
channel_id: 'UC_NO_MAPPING'
)
]
})

expect do
sync.sync!
end.to change(Post, :count).by(1)

post = Post.find_by!(url: 'https://www.youtube.com/watch?v=video-1')
tag_ids = post.tags.pluck(:id)

expect(post.title).to eq('YouTube テスト動画')
expect(post.uploaded_user_id).to be_nil
expect(post.original_created_from).to eq(Time.zone.parse('2026-05-01 12:34:00'))
expect(post.original_created_before).to eq(Time.zone.parse('2026-05-01 12:35:00'))

expect(tag_ids).to include(Tag.tagme.id)
expect(tag_ids).to include(Tag.bot.id)
expect(tag_ids).to include(Tag.youtube.id)
expect(tag_ids).to include(Tag.video.id)
expect(tag_ids).to include(Tag.no_deerjikist.id)

expect(PostVersionRecorder).to have_received(:record!).with(
post:,
event_type: :create,
created_by_user: nil
)
end

it 'uses deerjikist tag when channel id is mapped' do
Tag.tagme
Tag.bot
Tag.youtube
Tag.video
Tag.no_deerjikist

deerjikist_tag = Tag.find_or_create_by_tag_name!('テスト投稿者', category: :deerjikist)
Deerjikist.create!(
platform: 'youtube',
code: 'UC_MAPPED',
tag: deerjikist_tag
)

allow(sync).to receive(:query_terms).and_return([])
allow(sync).to receive(:playlist_ids).and_return(['PL123'])

allow(client).to receive(:playlist_items).with(
playlist_id: 'PL123',
page_token: nil
).and_return({
'items' => [
{
'contentDetails' => {
'videoId' => 'video-1'
}
}
]
})

allow(client).to receive(:videos).with(['video-1']).and_return({
'items' => [
youtube_video_item(
id: 'video-1',
title: 'YouTube テスト動画',
channel_id: 'UC_MAPPED'
)
]
})

sync.sync!

post = Post.find_by!(url: 'https://www.youtube.com/watch?v=video-1')
tag_ids = post.tags.pluck(:id)

expect(tag_ids).to include(deerjikist_tag.id)
expect(tag_ids).not_to include(Tag.no_deerjikist.id)
end

it 'removes no_deerjikist when deerjikist mapping is added later' do
Tag.no_deerjikist

post = Post.create!(
title: '旧タイトル',
url: 'https://www.youtube.com/watch?v=video-1',
uploaded_user_id: nil,
original_created_from: Time.zone.parse('2026-05-01 00:00:00'),
original_created_before: Time.zone.parse('2026-05-01 00:01:00')
)
PostTag.create!(post:, tag: Tag.no_deerjikist)

deerjikist_tag = Tag.find_or_create_by_tag_name!('後から判明した投稿者', category: :deerjikist)
Deerjikist.create!(
platform: 'youtube',
code: 'UC_MAPPED_LATER',
tag: deerjikist_tag
)

allow(sync).to receive(:query_terms).and_return([])
allow(sync).to receive(:playlist_ids).and_return(['PL123'])

allow(client).to receive(:playlist_items).with(
playlist_id: 'PL123',
page_token: nil
).and_return({
'items' => [
{
'contentDetails' => {
'videoId' => 'video-1'
}
}
]
})

allow(client).to receive(:videos).with(['video-1']).and_return({
'items' => [
youtube_video_item(
id: 'video-1',
title: '新タイトル',
channel_id: 'UC_MAPPED_LATER'
)
]
})

sync.sync!

post.reload
tag_ids = post.tags.pluck(:id)

expect(post.title).to eq('新タイトル')
expect(tag_ids).to include(deerjikist_tag.id)
expect(tag_ids).not_to include(Tag.no_deerjikist.id)

expect(PostVersionRecorder).to have_received(:ensure_snapshot!).with(
post,
created_by_user: nil
)
expect(PostVersionRecorder).to have_received(:record!).with(
post:,
event_type: :update,
created_by_user: nil
)
end

it 'matches existing youtu.be URL and does not create duplicate post' do
post = Post.create!(
title: '旧タイトル',
url: 'https://youtu.be/video-1',
uploaded_user_id: nil,
original_created_from: Time.zone.parse('2026-05-01 00:00:00'),
original_created_before: Time.zone.parse('2026-05-01 00:01:00')
)

allow(sync).to receive(:query_terms).and_return([])
allow(sync).to receive(:playlist_ids).and_return(['PL123'])

allow(client).to receive(:playlist_items).with(
playlist_id: 'PL123',
page_token: nil
).and_return({
'items' => [
{
'contentDetails' => {
'videoId' => 'video-1'
}
}
]
})

allow(client).to receive(:videos).with(['video-1']).and_return({
'items' => [
youtube_video_item(
id: 'video-1',
title: '新タイトル',
channel_id: 'UC_NO_MAPPING'
)
]
})

expect do
sync.sync!
end.not_to change(Post, :count)

expect(post.reload.title).to eq('新タイトル')
end
end

def youtube_video_item(id:, title:, channel_id:)
{
'id' => id,
'snippet' => {
'title' => title,
'channelId' => channel_id,
'publishedAt' => '2026-05-01T12:34:56Z',
'thumbnails' => {
'high' => {
'url' => "https://img.youtube.com/#{id}.jpg"
}
},
'tags' => ['tag-a', 'tag-b']
}
}
end
end

+ 93
- 0
backend/spec/services/youtube/video_item_spec.rb View File

@@ -0,0 +1,93 @@
require 'rails_helper'

RSpec.describe Youtube::VideoItem do
describe '#initialize' do
it 'extracts fields from YouTube video API item' do
item = {
'id' => 'video-1',
'snippet' => {
'title' => 'テスト動画',
'channelId' => 'UC123',
'publishedAt' => '2026-05-01T12:34:56Z',
'tags' => ['tag-a', 'tag-b'],
'thumbnails' => {
'high' => {
'url' => 'https://img.youtube.com/high.jpg'
},
'medium' => {
'url' => 'https://img.youtube.com/medium.jpg'
}
}
}
}

video = described_class.new(item)

expect(video.id).to eq('video-1')
expect(video.title).to eq('テスト動画')
expect(video.channel_id).to eq('UC123')
expect(video.published_at).to eq(Time.iso8601('2026-05-01T12:34:56Z'))
expect(video.thumbnail_url).to eq('https://img.youtube.com/high.jpg')
expect(video.raw_tags).to eq(['tag-a', 'tag-b'])
expect(video.url).to eq('https://www.youtube.com/watch?v=video-1')
end

it 'uses highest priority thumbnail' do
item = {
'id' => 'video-1',
'snippet' => {
'title' => 'テスト動画',
'channelId' => 'UC123',
'publishedAt' => '2026-05-01T12:34:56Z',
'thumbnails' => {
'default' => {
'url' => 'https://img.youtube.com/default.jpg'
},
'standard' => {
'url' => 'https://img.youtube.com/standard.jpg'
},
'maxres' => {
'url' => 'https://img.youtube.com/maxres.jpg'
}
}
}
}

video = described_class.new(item)

expect(video.thumbnail_url).to eq('https://img.youtube.com/maxres.jpg')
end

it 'falls back to empty raw tags' do
item = {
'id' => 'video-1',
'snippet' => {
'title' => 'テスト動画',
'channelId' => 'UC123',
'publishedAt' => '2026-05-01T12:34:56Z',
'thumbnails' => {}
}
}

video = described_class.new(item)

expect(video.raw_tags).to eq([])
end

it 'returns nil thumbnail when no thumbnail exists' do
item = {
'id' => 'video-1',
'snippet' => {
'title' => 'テスト動画',
'channelId' => 'UC123',
'publishedAt' => '2026-05-01T12:34:56Z',
'thumbnails' => {}
}
}

video = described_class.new(item)

expect(video.thumbnail_url).to be_nil
end
end
end

+ 25
- 0
backend/spec/tasks/post_sync_spec.rb View File

@@ -0,0 +1,25 @@
require 'rails_helper'
require 'rake'

RSpec.describe 'post:sync' do
around do |example|
original_application = Rake.application
Rake.application = Rake::Application.new

Rake::Task.define_task(:environment)
load Rails.root.join('lib/tasks/sync_posts.rake')

example.run
ensure
Rake.application = original_application
end

it 'runs Youtube::Sync' do
sync = instance_double(Youtube::Sync)

expect(Youtube::Sync).to receive(:new).once.and_return(sync)
expect(sync).to receive(:sync!).once

Rake::Task['post:sync'].invoke
end
end

Write Preview
Loading…
Cancel
Save