Merge remote-tracking branch 'origin/main' into feature/047

backend/app/services/nico_tag_version_recorder.rb

@@ -0,0 +1,19 @@
+class NicoTagVersionRecorder < VersionRecorder
+  def self.record! tag:, event_type:, created_by_user:
+    new(tag:, event_type:, created_by_user:).record!
+  end
+
+  def initialize tag:, event_type:, created_by_user:
+    super(record: tag, event_type:, created_by_user:)
+  end
+
+  private
+
+  def version_class = NicoTagVersion
+  def version_association = :nico_tag_versions
+  def record_key = :tag
+
+  def snapshot_attributes
+    { name: @record.name, linked_tags: @record.snapshot_linked_tag_names.join(' ') }
+  end
+end

backend/app/services/post_version_recorder.rb

@@ -0,0 +1,30 @@
+class PostVersionRecorder < VersionRecorder
+  def self.record! post:, event_type:, created_by_user:
+    new(post:, event_type:, created_by_user:).record!
+  end
+
+  def initialize post:, event_type:, created_by_user:
+    super(record: post, event_type:, created_by_user:)
+  end
+
+  def self.ensure_snapshot! post, created_by_user:
+    return if post.post_versions.exists?
+
+    record!(post:, event_type: :create, created_by_user:)
+  end
+
+  private
+
+  def version_class = PostVersion
+  def version_association = :post_versions
+  def record_key = :post
+
+  def snapshot_attributes
+    { title: @record.title,
+      url: @record.url,
+      thumbnail_base: @record.thumbnail_base,
+      tags: @record.snapshot_tag_names.join(' '),
+      original_created_from: @record.original_created_from,
+      original_created_before: @record.original_created_before }
+  end
+end

backend/app/services/tag_version_recorder.rb

@@ -0,0 +1,22 @@
+class TagVersionRecorder < VersionRecorder
+  def self.record! tag:, event_type:, created_by_user:
+    new(tag:, event_type:, created_by_user:).record!
+  end
+
+  def initialize tag:, event_type:, created_by_user:
+    super(record: tag, event_type:, created_by_user:)
+  end
+
+  private
+
+  def version_class = TagVersion
+  def version_association = :tag_versions
+  def record_key = :tag
+
+  def snapshot_attributes
+    { name: @record.name,
+      category: @record.category,
+      aliases: @record.snapshot_aliases.join(' '),
+      parent_tag_ids: @record.snapshot_parent_tag_ids.join(' ') }
+  end
+end

backend/app/services/tag_versioning.rb

@@ -0,0 +1,38 @@
+class TagVersioning
+  def self.record! tag, event_type:, created_by_user:
+    if tag.nico?
+      NicoTagVersionRecorder.record!(tag:, event_type:, created_by_user:)
+    else
+      TagVersionRecorder.record!(tag:, event_type:, created_by_user:)
+    end
+  end
+
+  def self.ensure_snapshot! tag, created_by_user:
+    if tag.nico?
+      return if tag.nico_tag_versions.exists?
+
+      NicoTagVersionRecorder.record!(tag:, event_type: :create, created_by_user:)
+    else
+      return if tag.tag_versions.exists?
+
+      TagVersionRecorder.record!(tag:, event_type: :create, created_by_user:)
+    end
+  end
+
+  def self.record_tag_snapshot! tag, created_by_user:
+    event_type =
+        if tag.nico?
+          tag.nico_tag_versions.exists? ? :update : :create
+        else
+          tag.tag_versions.exists? ? :update : :create
+        end
+
+    record!(tag, event_type:, created_by_user:)
+  end
+
+  def self.record_tag_snapshots! tags, created_by_user:
+    tags.each do |tag|
+      record_tag_snapshot!(tag, created_by_user:)
+    end
+  end
+end

backend/app/services/version_recorder.rb

@@ -0,0 +1,62 @@
+class VersionRecorder
+  EVENT_TYPES = ['create', 'update', 'discard', 'restore'].freeze
+
+  def initialize record:, event_type:, created_by_user:
+    @record = record
+    @event_type = event_type.to_s
+    @created_by_user = created_by_user
+
+    validate_event_type!
+  end
+
+  def record!
+    raise "#{ record_class.name } must be persisted" unless @record.persisted?
+
+    ApplicationRecord.transaction do
+      @record = record_class.unscoped.lock.find(@record.id)
+      latest = latest_version
+
+      if !(latest) && @event_type != 'create'
+        raise "#{ version_class.name } first event must be create"
+      end
+
+      if @event_type == 'create' && latest
+        raise "#{ version_class.name } create event already exists"
+      end
+
+      attrs = snapshot_attributes
+
+      return latest if @event_type == 'update' && latest && same_snapshot?(latest, attrs)
+
+      version_class.create!(base_attributes(latest).merge(record_key => @record).merge(attrs))
+    end
+  end
+
+  private
+
+  def latest_version = versions.order(version_no: :desc).first
+
+  def versions = @record.public_send(version_association)
+
+  def base_attributes latest
+    { version_no: (latest&.version_no || 0) + 1,
+      event_type: @event_type,
+      created_at: Time.current,
+      created_by_user: @created_by_user }
+  end
+
+  def same_snapshot?(version, attrs) = attrs.all? { |k, v| version.public_send(k) == v }
+
+  def validate_event_type!
+    return if EVENT_TYPES.include?(@event_type)
+
+    raise ArgumentError, "Invalid event_type: #{ @event_type }"
+  end
+
+  def version_class = raise NotImplementedError
+  def version_association = raise NotImplementedError
+  def record_key = raise NotImplementedError
+  def snapshot_attributes = raise NotImplementedError
+
+  def record_class = @record.class
+end

backend/app/services/wiki/commit.rb

@@ -7,6 +7,31 @@ module Wiki
       ;
     end
 
+    def self.create_content! tag_name:, body:, created_by_user:, message: nil
+      normalised = normalise_body(body)
+
+      page = WikiPage.new(tag_name:,
+                          body: normalised,
+                          created_user: created_by_user,
+                          updated_user: created_by_user)
+
+      if normalised.blank?
+        page.errors.add(:body, :blank)
+        raise ActiveRecord::RecordInvalid, page
+      end
+
+      ActiveRecord::Base.transaction do
+        page.save!
+
+        new(page:, created_user: created_by_user).content!(
+            body: normalised,
+            message:,
+            base_revision_id: nil)
+
+        page
+      end
+    end
+
     def self.content! page:, body:, created_user:, message: nil, base_revision_id: nil
       new(page:, created_user:).content!(body:, message:, base_revision_id:)
     end
@@ -21,7 +46,12 @@ module Wiki
     end
 
     def content! body:, message:, base_revision_id:
-      normalised = normalise_body(body)
+      normalised = self.class.normalise_body(body)
+      if normalised.blank?
+        @page.errors.add(:body, :blank)
+        raise ActiveRecord::RecordInvalid, @page
+      end
+
       lines = split_lines(normalised)
 
       line_shas = lines.map { |line| Digest::SHA256.hexdigest(line) }
@@ -37,10 +67,19 @@ module Wiki
           current_id = @page.wiki_revisions.maximum(:id)
           if current_id && current_id != base_revision_id.to_i
             raise Conflict,
-                "競合が発生してゐます（現在の Id.：#{ current_id }，ベース Id.：#{ base_revision_id }）．"
+                "競合が発生してゐます" +
+                "（現在の Id.：#{ current_id }，ベース Id.：#{ base_revision_id }）．"
           end
         end
 
+        @page.update!(body: normalised)
+
+        WikiVersionRecorder.record!(
+            page: @page,
+            event_type: @page.wiki_versions.exists? ? :update : :create,
+            reason: message,
+            created_by_user: @created_user)
+
         rev = WikiRevision.create!(
             wiki_page:        @page,
             base_revision_id:,
@@ -54,65 +93,45 @@ module Wiki
         rows = line_ids.each_with_index.map do |line_id, pos|
           { wiki_revision_id: rev.id, wiki_line_id: line_id, position: pos }
         end
-        WikiRevisionLine.insert_all!(rows)
+        WikiRevisionLine.insert_all!(rows) if rows.any?
 
         rev
       end
     end
 
-    def redirect! redirect_page:, message:, base_revision_id:
-      ActiveRecord::Base.transaction do
-        @page.lock!
+    def redirect!(redirect_page:, message:, base_revision_id:) = raise '廃止しました．'
 
-        if base_revision_id.present?
-          current_id = @page.wiki_revisions.maximum(:id)
-          if current_id && current_id != base_revision_id.to_i
-            raise Conflict,
-                "競合が発生してゐます（現在の Id.：#{ current_id }，ベース Id.：#{ base_revision_id }）．"
-          end
-        end
-
-        WikiRevision.create!(
-            wiki_page:        @page,
-            base_revision_id:,
-            created_user:     @created_user,
-            kind:             :redirect,
-            redirect_page:,
-            message:,
-            lines_count:      0,
-            tree_sha256:      nil)
-      end
+    def self.normalise_body body
+      s = body.to_s
+      s.gsub!(/\r\n?/, "\n")
+      s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '🖕')
+      s.gsub(/\n+$/, '')
     end
 
     private
 
-    def normalise_body body
-      s = body.to_s
-      s.gsub!("\r\n", "\n")
-      s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '🖕')
-    end
-
-    def split_lines body
-      body.split("\n")
-    end
+    def split_lines(body) = body.split("\n")
 
     def upsert_lines! lines, line_shas
       now = Time.current
 
       id_by_sha = WikiLine.where(sha256: line_shas).pluck(:sha256, :id).to_h
 
-      missing_rows = []
+      missing_by_sha = { }
+
       line_shas.each_with_index do |sha, i|
         next if id_by_sha.key?(sha)
+        next if missing_by_sha.key?(sha)
 
-        missing_rows << { sha256:     sha,
-                          body:       lines[i],
-                          created_at: now,
-                          updated_at: now }
+        missing_by_sha[sha] = {
+            sha256:     sha,
+            body:       lines[i],
+            created_at: now,
+            updated_at: now }
       end
 
-      if missing_rows.any?
-        WikiLine.upsert_all(missing_rows)
+      if missing_by_sha.any?
+        WikiLine.upsert_all(missing_by_sha.values)
         id_by_sha = WikiLine.where(sha256: line_shas).pluck(:sha256, :id).to_h
       end
 

backend/app/services/wiki_version_recorder.rb

@@ -0,0 +1,21 @@
+class WikiVersionRecorder < VersionRecorder
+  def self.record! page:, event_type:, reason: nil, created_by_user:
+    new(page:, event_type:, reason:, created_by_user:).record!
+  end
+
+  def initialize page:, event_type:, reason: nil, created_by_user:
+    @reason = reason
+    super(record: page, event_type:, created_by_user:)
+  end
+
+  private
+
+  def version_class = WikiVersion
+  def version_association = :wiki_versions
+  def record_key = :wiki_page
+
+  def snapshot_attributes = {
+      title: @record.title,
+      body: @record.body,
+      reason: @reason }
+end

backend/app/services/youtube/api_client.rb

@@ -0,0 +1,73 @@
+require 'json'
+require 'net/http'
+require 'uri'
+
+
+module Youtube
+  class ApiClient
+    ENDPOINT = 'https://www.googleapis.com/youtube/v3'
+
+    def initialize api_key: ENV.fetch('YOUTUBE_API_KEY')
+      @api_key = api_key
+    end
+
+    def search_videos q:, published_after: nil, published_before: nil, page_token: nil
+      get_json('/search', {
+          part:           'snippet',
+          type:           'video',
+          q:,
+          order:          'date',
+          maxResults:     50,
+          regionCode:     'JP',
+          relevanceLanguage: 'ja',
+          publishedAfter: published_after&.iso8601,
+          publishedBefore: published_before&.iso8601,
+          pageToken:      page_token }.compact)
+    end
+
+    def videos ids
+      return { 'items' => [] } if ids.empty?
+
+      get_json('/videos', part: 'snippet,status,contentDetails', id: ids.join(','))
+    end
+
+    def playlist_items playlist_id:, page_token: nil
+      get_json('/playlistItems', {
+          part:       'snippet,contentDetails,status',
+          playlistId: playlist_id,
+          maxResults: 50,
+          pageToken:  page_token }.compact)
+    end
+
+    def channel id: nil, handle: nil
+      raise ArgumentError, 'id or handle is required' if id.present? == handle.present?
+
+      params = { part: 'snippet,contentDetails' }
+      params[:id] = id if id.present?
+      params[:forHandle] = handle if handle.present?
+
+      get_json('/channels', params)
+    end
+
+    private
+
+    def get_json path, params
+      uri = URI(ENDPOINT + path)
+      uri.query = URI.encode_www_form(params.merge(key: @api_key))
+
+      response = Net::HTTP.start(uri.host,
+                                 uri.port,
+                                 use_ssl: true,
+                                 open_timeout: 10,
+                                 read_timeout: 30) do |http|
+        http.get(uri)
+      end
+
+      unless response.is_a?(Net::HTTPSuccess)
+        raise "YouTube API error: #{ response.code } #{ response.body }"
+      end
+
+      JSON.parse(response.body)
+    end
+  end
+end

backend/app/services/youtube/sync.rb

@@ -0,0 +1,168 @@
+require 'open-uri'
+require 'set'
+require 'time'
+
+
+module Youtube
+  class Sync
+    def initialize client: ApiClient.new
+      @client = client
+    end
+
+    def sync!
+      video_ids = discover_video_ids
+      return if video_ids.empty?
+
+      video_ids.each_slice(50) do |ids|
+        @client.videos(ids).fetch('items', []).each do |item|
+          sync_video!(VideoItem.new(item))
+        end
+      end
+    end
+
+    private
+
+    def discover_video_ids
+      ids = Set.new
+
+      query_terms.each do |q|
+        response = @client.search_videos(q:, published_after: sync_since)
+
+        response.fetch('items', []).each do |item|
+          video_id = item.dig('id', 'videoId')
+          ids << video_id if video_id.present?
+        end
+      end
+
+      playlist_ids.each do |playlist_id|
+        each_playlist_item(playlist_id) do |item|
+          video_id = item.dig('contentDetails', 'videoId')
+          video_id ||= item.dig('snippet', 'resourceId', 'videoId')
+
+          ids << video_id if video_id.present?
+        end
+      end
+
+      ids.to_a
+    end
+
+    def sync_video! video
+      post = Post.where('url REGEXP ?', youtube_url_regexp(video.id)).first
+
+      original_created_from = video.published_at.change(sec: 0)
+      original_created_before = original_created_from + 1.minute
+
+      post_created = false
+      post_changed = false
+
+      if post
+        post.assign_attributes(title: video.title,
+                               original_created_from:,
+                               original_created_before:,
+                               thumbnail_base: video.thumbnail_url)
+
+        post_changed = post.changed?
+        post.save! if post_changed
+
+        attach_thumbnail_if_needed!(post, video.thumbnail_url)
+      else
+        post_created = true
+        post = Post.create!(
+            title: video.title,
+            url: video.url,
+            thumbnail_base: video.thumbnail_url,
+            uploaded_user_id: nil,
+            original_created_from:,
+            original_created_before:)
+
+        attach_thumbnail_if_needed!(post, video.thumbnail_url)
+
+        sync_post_tags!(post, [Tag.tagme.id, Tag.bot.id, Tag.youtube.id, Tag.video.id])
+      end
+
+      kept_tag_ids = post.tags.pluck(:id).to_set
+      desired_tag_ids = kept_tag_ids.to_a
+
+      deerjikist = Deerjikist.find_by(platform: :youtube, code: video.channel_id)
+      if deerjikist
+        desired_tag_ids.delete(Tag.no_deerjikist.id)
+        desired_tag_ids << deerjikist.tag_id
+      elsif post.tags.where(category: :deerjikist).none?
+        desired_tag_ids << Tag.no_deerjikist.id
+      end
+
+      desired_tag_ids.uniq!
+
+      sync_post_tags!(post, desired_tag_ids, current_tag_ids: kept_tag_ids)
+
+      if post_created
+        PostVersionRecorder.record!(post:, event_type: :create, created_by_user: nil)
+      elsif post_changed || kept_tag_ids != desired_tag_ids.to_set
+        PostVersionRecorder.ensure_snapshot!(post, created_by_user: nil)
+        PostVersionRecorder.record!(post:, event_type: :update, created_by_user: nil)
+      end
+    end
+
+    def sync_post_tags! post, desired_tag_ids, current_tag_ids: nil
+      current_tag_ids ||= PostTag.kept.where(post_id: post.id).pluck(:tag_id).to_set
+      desired_tag_ids = desired_tag_ids.compact.to_set
+
+      to_add = desired_tag_ids - current_tag_ids
+      to_remove = current_tag_ids - desired_tag_ids
+
+      Tag.where(id: to_add.to_a).find_each do |tag|
+        begin
+          PostTag.create!(post:, tag:)
+        rescue ActiveRecord::RecordNotUnique
+          ;
+        end
+      end
+
+      PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each do |pt|
+        pt.discard_by!(nil)
+      end
+    end
+
+    def attach_thumbnail_if_needed! post, thumbnail_url
+      return if post.thumbnail.attached?
+      return if thumbnail_url.blank?
+
+      post.thumbnail.attach(
+          io:           URI.open(thumbnail_url),
+          filename:     File.basename(URI.parse(thumbnail_url).path),
+          content_type: 'image/jpeg')
+
+      post.resized_thumbnail!
+    end
+
+    def youtube_url_regexp id
+      escaped = Regexp.escape(id)
+      "(youtube\\.com/watch\\?v=#{ escaped }|youtu\\.be/#{ escaped })([^A-Za-z0-9_-]|$)"
+    end
+
+    def query_terms = ['ぼざろクリーチャーシリーズ', '伊地知ニジカ', '伊地知虹鹿']
+
+    def playlist_ids
+      ['PLrOch4zHkI5vu29b-f9umUQQ4tQkuWLPX',
+       'PLrOch4zHkI5vOK0RaytQq6PbucxQkkL0K',
+       'PLrOch4zHkI5tdwm9vSegiDQJOM-hgpcOC']
+    end
+
+    def sync_since = 14.days.ago
+
+    def each_playlist_item playlist_id
+      page_token = nil
+
+      loop do
+        response = @client.playlist_items(playlist_id:, page_token:)
+
+        response.fetch('items', []).each do |item|
+          yield item
+        end
+
+        page_token = response['nextPageToken']
+        break if page_token.blank?
+      end
+    end
+  end
+end

backend/app/services/youtube/video_item.rb

@@ -0,0 +1,32 @@
+require 'time'
+
+
+module Youtube
+  class VideoItem
+    attr_reader :id, :title, :channel_id, :published_at, :thumbnail_url, :raw_tags
+
+    def initialize item
+      snippet = item.fetch('snippet')
+
+      @id = item.fetch('id')
+      @title = snippet['title']
+      @channel_id = snippet['channelId']
+      @published_at = Time.iso8601(snippet['publishedAt'])
+      @thumbnail_url = pick_thumbnail(snippet['thumbnails'] || { })
+      @raw_tags = snippet['tags'] || []
+    end
+
+    def url = "https://www.youtube.com/watch?v=#{ @id }"
+
+    private
+
+    def pick_thumbnail thumbnails
+      ['maxres', 'standard', 'high', 'medium', 'default'].each do |key|
+        url = thumbnails.dig(key, 'url')
+        return url if url.present?
+      end
+
+      nil
+    end
+  end
+end