feature/084 into main 1 day ago
| @@ -63,3 +63,5 @@ gem 'diff-lcs' | |||
| gem 'dotenv-rails' | |||
| gem 'whenever', require: false | |||
| gem 'discard' | |||
| @@ -90,6 +90,8 @@ GEM | |||
| crass (1.0.6) | |||
| date (3.4.1) | |||
| diff-lcs (1.6.2) | |||
| discard (1.4.0) | |||
| activerecord (>= 4.2, < 9.0) | |||
| dotenv (3.1.8) | |||
| dotenv-rails (3.1.8) | |||
| dotenv (= 3.1.8) | |||
| @@ -420,6 +422,7 @@ DEPENDENCIES | |||
| bootsnap | |||
| brakeman | |||
| diff-lcs | |||
| discard | |||
| dotenv-rails | |||
| gollum | |||
| image_processing (~> 1.14) | |||
| @@ -1,7 +1,3 @@ | |||
| require 'open-uri' | |||
| require 'nokogiri' | |||
| class PostsController < ApplicationController | |||
| # GET /posts | |||
| def index | |||
| @@ -80,8 +76,9 @@ class PostsController < ApplicationController | |||
| post.thumbnail.attach(thumbnail) | |||
| if post.save | |||
| post.resized_thumbnail! | |||
| post.tags = Tag.normalise_tags(tag_names) | |||
| post.tags = Tag.expand_parent_tags(post.tags) | |||
| tags = Tag.normalise_tags(tag_names) | |||
| tags = Tag.expand_parent_tags(tags) | |||
| sync_post_tags!(post, tags) | |||
| render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }), | |||
| status: :created | |||
| else | |||
| @@ -114,10 +111,11 @@ class PostsController < ApplicationController | |||
| original_created_before = params[:original_created_before] | |||
| post = Post.find(params[:id].to_i) | |||
| if post.update(title:, original_created_from:, original_created_before:) | |||
| tags = post.tags.where(category: 'nico').to_a + | |||
| Tag.normalise_tags(tag_names, with_tagme: false) | |||
| tags = Tag.expand_parent_tags(tags) | |||
| if post.update(title:, tags:, original_created_from:, original_created_before:) | |||
| sync_post_tags!(post, tags) | |||
| json = post.as_json | |||
| json['tags'] = build_tag_tree_for(post.tags) | |||
| render json:, status: :ok | |||
| @@ -135,7 +133,11 @@ class PostsController < ApplicationController | |||
| def filtered_posts | |||
| tag_names = params[:tags]&.split(' ') | |||
| match_type = params[:match] | |||
| tag_names.present? ? filter_posts_by_tags(tag_names, match_type) : Post.all | |||
| if tag_names.present? | |||
| filter_posts_by_tags(tag_names, match_type) | |||
| else | |||
| Post.all | |||
| end | |||
| end | |||
| def filter_posts_by_tags tag_names, match_type | |||
| @@ -150,6 +152,30 @@ class PostsController < ApplicationController | |||
| posts.distinct | |||
| end | |||
| def sync_post_tags! post, desired_tags | |||
| desired_tags.each do |t| | |||
| t.save! if t.new_record? | |||
| end | |||
| desired_ids = desired_tags.map(&:id).to_set | |||
| current_ids = post.tags.pluck(:id).to_set | |||
| to_add = desired_ids - current_ids | |||
| to_remove = current_ids - desired_ids | |||
| Tag.where(id: to_add).find_each do |tag| | |||
| begin | |||
| PostTag.create!(post:, tag:, created_user: current_user) | |||
| rescue ActiveRecord::RecordNotUnique | |||
| ; | |||
| end | |||
| end | |||
| PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each do |pt| | |||
| pt.discard_by!(current_user) | |||
| end | |||
| end | |||
| def build_tag_tree_for tags | |||
| tags = tags.to_a | |||
| tag_ids = tags.map(&:id) | |||
| @@ -1,11 +1,13 @@ | |||
| class Post < ApplicationRecord | |||
| require 'mini_magick' | |||
| class Post < ApplicationRecord | |||
| belongs_to :parent, class_name: 'Post', optional: true, foreign_key: 'parent_id' | |||
| belongs_to :uploaded_user, class_name: 'User', optional: true | |||
| has_many :post_tags, dependent: :destroy | |||
| has_many :tags, through: :post_tags | |||
| has_many :post_tags, dependent: :destroy, inverse_of: :post | |||
| has_many :active_post_tags, -> { kept }, class_name: 'PostTag', inverse_of: :post | |||
| has_many :post_tags_with_discarded, -> { with_discarded }, class_name: 'PostTag' | |||
| has_many :tags, through: :active_post_tags | |||
| has_many :user_post_views, dependent: :destroy | |||
| has_many :post_similarities_as_post, | |||
| class_name: 'PostSimilarity', | |||
| @@ -1,7 +1,25 @@ | |||
| class PostTag < ApplicationRecord | |||
| include Discard::Model | |||
| belongs_to :post | |||
| belongs_to :tag, counter_cache: :post_count | |||
| belongs_to :created_user, class_name: 'User', optional: true | |||
| belongs_to :deleted_user, class_name: 'User', optional: true | |||
| validates :post_id, presence: true | |||
| validates :tag_id, presence: true | |||
| validates :post_id, uniqueness: { | |||
| scope: :tag_id, | |||
| conditions: -> { where(discarded_at: nil) } } | |||
| def discard_by! deleted_user | |||
| return self if discarded? | |||
| transaction do | |||
| update!(discarded_at: Time.current, deleted_user:) | |||
| Tag.where(id: tag_id).update_all('post_count = GREATEST(post_count - 1, 0)') | |||
| end | |||
| self | |||
| end | |||
| end | |||
| @@ -1,6 +1,8 @@ | |||
| class Tag < ApplicationRecord | |||
| has_many :post_tags, dependent: :destroy | |||
| has_many :posts, through: :post_tags | |||
| has_many :post_tags, dependent: :delete_all, inverse_of: :tag | |||
| has_many :active_post_tags, -> { kept }, class_name: 'PostTag', inverse_of: :tag | |||
| has_many :post_tags_with_discarded, -> { with_discarded }, class_name: 'PostTag' | |||
| has_many :posts, through: :active_post_tags | |||
| has_many :tag_aliases, dependent: :destroy | |||
| has_many :nico_tag_relations, foreign_key: :nico_tag_id, dependent: :destroy | |||
| @@ -43,13 +45,13 @@ class Tag < ApplicationRecord | |||
| 'meta:' => 'meta' }.freeze | |||
| def self.tagme | |||
| @tagme ||= Tag.find_or_initialize_by(name: 'タグ希望') do |tag| | |||
| @tagme ||= Tag.find_or_create_by!(name: 'タグ希望') do |tag| | |||
| tag.category = 'meta' | |||
| end | |||
| end | |||
| def self.bot | |||
| @bot ||= Tag.find_or_initialize_by(name: 'bot操作') do |tag| | |||
| @bot ||= Tag.find_or_create_by!(name: 'bot操作') do |tag| | |||
| tag.category = 'meta' | |||
| end | |||
| end | |||
| @@ -0,0 +1,36 @@ | |||
| class AddDiscardToPostTags < ActiveRecord::Migration[8.0] | |||
| def up | |||
| execute <<~SQL | |||
| DELETE | |||
| pt1 | |||
| FROM | |||
| post_tags pt1 | |||
| INNER JOIN | |||
| post_tags pt2 | |||
| ON | |||
| pt1.post_id = pt2.post_id | |||
| AND pt1.tag_id = pt2.tag_id | |||
| AND pt1.id > pt2.id | |||
| ; | |||
| SQL | |||
| add_column :post_tags, :discarded_at, :datetime | |||
| add_index :post_tags, :discarded_at | |||
| add_column :post_tags, :is_active, :boolean, | |||
| as: 'discarded_at IS NULL', stored: true | |||
| add_column :post_tags, :active_unique_key, :string, | |||
| as: "CASE WHEN discarded_at IS NULL THEN CONCAT(post_id, ':', tag_id) ELSE NULL END", | |||
| stored: true | |||
| add_index :post_tags, :active_unique_key, unique: true, name: 'idx_post_tags_active_unique' | |||
| add_index :post_tags, [:post_id, :discarded_at] | |||
| add_index :post_tags, [:tag_id, :discarded_at] | |||
| end | |||
| def down | |||
| raise ActiveRecord::IrreversibleMigration, '戻せません.' | |||
| end | |||
| end | |||
| @@ -5,12 +5,30 @@ namespace :nico do | |||
| require 'open-uri' | |||
| require 'nokogiri' | |||
| fetch_thumbnail = -> url { | |||
| fetch_thumbnail = -> url do | |||
| html = URI.open(url, read_timeout: 60, 'User-Agent' => 'Mozilla/5.0').read | |||
| doc = Nokogiri::HTML(html) | |||
| doc.at('meta[name="thumbnail"]')&.[]('content').presence | |||
| } | |||
| end | |||
| def sync_post_tags! post, desired_tag_ids | |||
| desired_ids = desired_tag_ids.compact.to_set | |||
| current_ids = post.tags.pluck(:id).to_set | |||
| to_add = desired_ids - current_ids | |||
| to_remove = current_ids - desired_ids | |||
| Tag.where(id: to_add.to_a).find_each do |tag| | |||
| begin | |||
| PostTag.create!(post:, tag:) | |||
| rescue ActiveRecord::RecordNotUnique | |||
| ; | |||
| end | |||
| end | |||
| PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each(&:discard!) | |||
| end | |||
| mysql_user = ENV['MYSQL_USER'] | |||
| mysql_pass = ENV['MYSQL_PASS'] | |||
| @@ -19,7 +37,8 @@ namespace :nico do | |||
| { 'MYSQL_USER' => mysql_user, 'MYSQL_PASS' => mysql_pass }, | |||
| 'python3', "#{ nizika_nico_path }/get_videos.py") | |||
| if status.success? | |||
| abort unless status.success? | |||
| data = JSON.parse(stdout) | |||
| data.each do |datum| | |||
| post = Post.where('url LIKE ?', '%nicovideo.jp%').find { |post| | |||
| @@ -40,22 +59,31 @@ namespace :nico do | |||
| post.resized_thumbnail! | |||
| end | |||
| current_tags = post.tags.where(category: 'nico').pluck(:name).sort | |||
| new_tags = datum['tags'].map { |tag| "nico:#{ tag }" }.sort | |||
| if current_tags != new_tags | |||
| post.tags.destroy(post.tags.where(name: current_tags)) | |||
| tags_to_add = [] | |||
| new_tags.each do |name| | |||
| kept_tags = post.tags.reload | |||
| kept_non_nico_ids = kept_tags.where.not(category: 'nico').pluck(:id).to_set | |||
| desired_nico_ids = [] | |||
| datum['tags'].each do |raw| | |||
| name = "nico:#{ raw }" | |||
| tag = Tag.find_or_initialize_by(name:) do |t| | |||
| t.category = 'nico' | |||
| end | |||
| tags_to_add.concat([tag] + tag.linked_tags) | |||
| end | |||
| tags_to_add << Tag.tagme if post.tags.size < 10 | |||
| tags_to_add << Tag.bot | |||
| post.tags = (post.tags + tags_to_add).uniq | |||
| end | |||
| tag.save! if tag.new_record? | |||
| desired_nico_ids << tag.id | |||
| desired_nico_ids.concat(tag.linked_tags.pluck(:id)) | |||
| end | |||
| desired_nico_ids.uniq! | |||
| desired_extra_ids = [] | |||
| desired_extra_ids << Tag.tagme.id if kept_tags.size < 10 | |||
| desired_extra_ids << Tag.bot.id | |||
| desired_extra_ids.compact! | |||
| desired_extra_ids.uniq! | |||
| desired_all_ids = kept_non_nico_ids.to_a + desired_nico_ids + desired_extra_ids | |||
| desired_all_ids.uniq! | |||
| sync_post_tags!(post, desired_all_ids) | |||
| end | |||
| end | |||
| end | |||