| @@ -63,3 +63,5 @@ gem 'diff-lcs' | |||||
| gem 'dotenv-rails' | gem 'dotenv-rails' | ||||
| gem 'whenever', require: false | gem 'whenever', require: false | ||||
| gem 'discard' | |||||
| @@ -90,6 +90,8 @@ GEM | |||||
| crass (1.0.6) | crass (1.0.6) | ||||
| date (3.4.1) | date (3.4.1) | ||||
| diff-lcs (1.6.2) | diff-lcs (1.6.2) | ||||
| discard (1.4.0) | |||||
| activerecord (>= 4.2, < 9.0) | |||||
| dotenv (3.1.8) | dotenv (3.1.8) | ||||
| dotenv-rails (3.1.8) | dotenv-rails (3.1.8) | ||||
| dotenv (= 3.1.8) | dotenv (= 3.1.8) | ||||
| @@ -420,6 +422,7 @@ DEPENDENCIES | |||||
| bootsnap | bootsnap | ||||
| brakeman | brakeman | ||||
| diff-lcs | diff-lcs | ||||
| discard | |||||
| dotenv-rails | dotenv-rails | ||||
| gollum | gollum | ||||
| image_processing (~> 1.14) | image_processing (~> 1.14) | ||||
| @@ -1,7 +1,3 @@ | |||||
| require 'open-uri' | |||||
| require 'nokogiri' | |||||
| class PostsController < ApplicationController | class PostsController < ApplicationController | ||||
| # GET /posts | # GET /posts | ||||
| def index | def index | ||||
| @@ -77,7 +73,7 @@ class PostsController < ApplicationController | |||||
| post.thumbnail.attach(thumbnail) | post.thumbnail.attach(thumbnail) | ||||
| if post.save | if post.save | ||||
| post.resized_thumbnail! | post.resized_thumbnail! | ||||
| post.tags = Tag.normalise_tags(tag_names) | |||||
| sync_post_tags!(post, Tag.normalise_tags(tag_names)) | |||||
| render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }), | render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }), | ||||
| status: :created | status: :created | ||||
| else | else | ||||
| @@ -110,8 +106,10 @@ class PostsController < ApplicationController | |||||
| original_created_before = params[:original_created_before] | original_created_before = params[:original_created_before] | ||||
| post = Post.find(params[:id].to_i) | post = Post.find(params[:id].to_i) | ||||
| tags = post.tags.where(category: 'nico').to_a + Tag.normalise_tags(tag_names) | |||||
| if post.update(title:, tags:, original_created_from:, original_created_before:) | |||||
| if post.update(title:, original_created_from:, original_created_before:) | |||||
| sync_post_tags!(post, | |||||
| (post.tags.where(category: 'nico').to_a + | |||||
| Tag.normalise_tags(tag_names))) | |||||
| render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }), | render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }), | ||||
| status: :ok | status: :ok | ||||
| else | else | ||||
| @@ -128,7 +126,11 @@ class PostsController < ApplicationController | |||||
| def filtered_posts | def filtered_posts | ||||
| tag_names = params[:tags]&.split(' ') | tag_names = params[:tags]&.split(' ') | ||||
| match_type = params[:match] | match_type = params[:match] | ||||
| tag_names.present? ? filter_posts_by_tags(tag_names, match_type) : Post.all | |||||
| if tag_names.present? | |||||
| filter_posts_by_tags(tag_names, match_type) | |||||
| else | |||||
| Post.all | |||||
| end | |||||
| end | end | ||||
| def filter_posts_by_tags tag_names, match_type | def filter_posts_by_tags tag_names, match_type | ||||
| @@ -142,4 +144,28 @@ class PostsController < ApplicationController | |||||
| end | end | ||||
| posts.distinct | posts.distinct | ||||
| end | end | ||||
| def sync_post_tags! post, desired_tags | |||||
| desired_tags.each do |t| | |||||
| t.save! if t.new_record? | |||||
| end | |||||
| desired_ids = desired_tags.map(&:id).to_set | |||||
| current_ids = post.tags.pluck(:id).to_set | |||||
| to_add = desired_ids - current_ids | |||||
| to_remove = current_ids - desired_ids | |||||
| Tag.where(id: to_add).find_each do |tag| | |||||
| begin | |||||
| PostTag.create!(post:, tag:, created_user: current_user) | |||||
| rescue ActiveRecord::RecordNotUnique | |||||
| ; | |||||
| end | |||||
| end | |||||
| PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each do |pt| | |||||
| pt.discard_by!(current_user) | |||||
| end | |||||
| end | |||||
| end | end | ||||
| @@ -1,11 +1,13 @@ | |||||
| require 'mini_magick' | |||||
| class Post < ApplicationRecord | class Post < ApplicationRecord | ||||
| require 'mini_magick' | |||||
| belongs_to :parent, class_name: 'Post', optional: true, foreign_key: 'parent_id' | belongs_to :parent, class_name: 'Post', optional: true, foreign_key: 'parent_id' | ||||
| belongs_to :uploaded_user, class_name: 'User', optional: true | belongs_to :uploaded_user, class_name: 'User', optional: true | ||||
| has_many :post_tags, dependent: :destroy | |||||
| has_many :tags, through: :post_tags | |||||
| has_many :post_tags, dependent: :delete_all, inverse_of: :post | |||||
| has_many :active_post_tags, -> { kept }, class_name: 'PostTag', inverse_of: :post | |||||
| has_many :post_tags_with_discarded, -> { with_discarded }, class_name: 'PostTag' | |||||
| has_many :tags, through: :active_post_tags | |||||
| has_many :user_post_views, dependent: :destroy | has_many :user_post_views, dependent: :destroy | ||||
| has_many :post_similarities_as_post, | has_many :post_similarities_as_post, | ||||
| class_name: 'PostSimilarity', | class_name: 'PostSimilarity', | ||||
| @@ -1,7 +1,25 @@ | |||||
| class PostTag < ApplicationRecord | class PostTag < ApplicationRecord | ||||
| include Discard::Model | |||||
| belongs_to :post | belongs_to :post | ||||
| belongs_to :tag, counter_cache: :post_count | belongs_to :tag, counter_cache: :post_count | ||||
| belongs_to :created_user, class_name: 'User', optional: true | |||||
| belongs_to :deleted_user, class_name: 'User', optional: true | |||||
| validates :post_id, presence: true | validates :post_id, presence: true | ||||
| validates :tag_id, presence: true | validates :tag_id, presence: true | ||||
| validates :post_id, uniqueness: { | |||||
| scope: :tag_id, | |||||
| conditions: -> { where(discarded_at: nil) } } | |||||
| def discard_by! deleted_user | |||||
| return self if discarded? | |||||
| transaction do | |||||
| update!(discarded_at: Time.current, deleted_user:) | |||||
| Tag.where(id: tag_id).update_all('post_count = GREATEST(post_count - 1, 0)') | |||||
| end | |||||
| self | |||||
| end | |||||
| end | end | ||||
| @@ -1,6 +1,8 @@ | |||||
| class Tag < ApplicationRecord | class Tag < ApplicationRecord | ||||
| has_many :post_tags, dependent: :destroy | |||||
| has_many :posts, through: :post_tags | |||||
| has_many :post_tags, dependent: :delete_all, inverse_of: :tag | |||||
| has_many :active_post_tags, -> { kept }, class_name: 'PostTag', :inverse_of: :tag | |||||
| has_many :post_tags_with_discarded, -> { with_discarded }, class_name: 'PostTag' | |||||
| has_many :posts, through: :active_post_tags | |||||
| has_many :tag_aliases, dependent: :destroy | has_many :tag_aliases, dependent: :destroy | ||||
| has_many :nico_tag_relations, foreign_key: :nico_tag_id, dependent: :destroy | has_many :nico_tag_relations, foreign_key: :nico_tag_id, dependent: :destroy | ||||
| @@ -35,13 +37,13 @@ class Tag < ApplicationRecord | |||||
| 'meta:' => 'meta' }.freeze | 'meta:' => 'meta' }.freeze | ||||
| def self.tagme | def self.tagme | ||||
| @tagme ||= Tag.find_or_initialize_by(name: 'タグ希望') do |tag| | |||||
| @tagme ||= Tag.find_or_create_by!(name: 'タグ希望') do |tag| | |||||
| tag.category = 'meta' | tag.category = 'meta' | ||||
| end | end | ||||
| end | end | ||||
| def self.bot | def self.bot | ||||
| @bot ||= Tag.find_or_initialize_by(name: 'bot操作') do |tag| | |||||
| @bot ||= Tag.find_or_create_by!(name: 'bot操作') do |tag| | |||||
| tag.category = 'meta' | tag.category = 'meta' | ||||
| end | end | ||||
| end | end | ||||
| @@ -0,0 +1,18 @@ | |||||
| class AddDiscardToPostTags < ActiveRecord::Migration[8.0] | |||||
| def change | |||||
| add_column :post_tags, :discarded_at, :datetime | |||||
| add_index :post_tags, :discarded_at | |||||
| add_column :post_tags, :is_active, :boolean, | |||||
| as: 'discarded_at IS NULL', stored: true | |||||
| add_column :post_tags, :active_unique_key, :string, | |||||
| as: "CASE WHEN discarded_at IS NULL THEN CONCAT(post_id, ':', tag_id) ELSE NULL END", | |||||
| stored: true | |||||
| add_index :post_tags, :active_unique_key, unique: true, name: 'idx_post_tags_active_unique' | |||||
| add_index :post_tags, [:post_id, :discarded_at] | |||||
| add_index :post_tags, [:tag_id, :discarded_at] | |||||
| end | |||||
| end | |||||
| @@ -10,7 +10,7 @@ | |||||
| # | # | ||||
| # It's strongly recommended that you check this file into your version control system. | # It's strongly recommended that you check this file into your version control system. | ||||
| ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do | |||||
| ActiveRecord::Schema[8.0].define(version: 2025_10_11_200300) do | |||||
| create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| | create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| | ||||
| t.string "name", null: false | t.string "name", null: false | ||||
| t.string "record_type", null: false | t.string "record_type", null: false | ||||
| @@ -70,9 +70,16 @@ ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do | |||||
| t.bigint "deleted_user_id" | t.bigint "deleted_user_id" | ||||
| t.datetime "created_at", null: false | t.datetime "created_at", null: false | ||||
| t.datetime "updated_at", null: false | t.datetime "updated_at", null: false | ||||
| t.datetime "discarded_at" | |||||
| t.virtual "is_active", type: :boolean, as: "(`discarded_at` is null)", stored: true | |||||
| t.virtual "active_unique_key", type: :string, as: "(case when (`discarded_at` is null) then concat(`post_id`,_utf8mb4':',`tag_id`) else NULL end)", stored: true | |||||
| t.index ["active_unique_key"], name: "idx_post_tags_active_unique", unique: true | |||||
| t.index ["created_user_id"], name: "index_post_tags_on_created_user_id" | t.index ["created_user_id"], name: "index_post_tags_on_created_user_id" | ||||
| t.index ["deleted_user_id"], name: "index_post_tags_on_deleted_user_id" | t.index ["deleted_user_id"], name: "index_post_tags_on_deleted_user_id" | ||||
| t.index ["discarded_at"], name: "index_post_tags_on_discarded_at" | |||||
| t.index ["post_id", "discarded_at"], name: "index_post_tags_on_post_id_and_discarded_at" | |||||
| t.index ["post_id"], name: "index_post_tags_on_post_id" | t.index ["post_id"], name: "index_post_tags_on_post_id" | ||||
| t.index ["tag_id", "discarded_at"], name: "index_post_tags_on_tag_id_and_discarded_at" | |||||
| t.index ["tag_id"], name: "index_post_tags_on_tag_id" | t.index ["tag_id"], name: "index_post_tags_on_tag_id" | ||||
| end | end | ||||
| @@ -107,6 +114,15 @@ ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do | |||||
| t.index ["tag_id"], name: "index_tag_aliases_on_tag_id" | t.index ["tag_id"], name: "index_tag_aliases_on_tag_id" | ||||
| end | end | ||||
| create_table "tag_implications", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| | |||||
| t.bigint "tag_id", null: false | |||||
| t.bigint "parent_tag_id", null: false | |||||
| t.datetime "created_at", null: false | |||||
| t.datetime "updated_at", null: false | |||||
| t.index ["parent_tag_id"], name: "index_tag_implications_on_parent_tag_id" | |||||
| t.index ["tag_id"], name: "index_tag_implications_on_tag_id" | |||||
| end | |||||
| create_table "tag_similarities", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| | create_table "tag_similarities", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| | ||||
| t.bigint "tag_id", null: false | t.bigint "tag_id", null: false | ||||
| t.bigint "target_tag_id", null: false | t.bigint "target_tag_id", null: false | ||||
| @@ -176,6 +192,8 @@ ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do | |||||
| add_foreign_key "posts", "users", column: "uploaded_user_id" | add_foreign_key "posts", "users", column: "uploaded_user_id" | ||||
| add_foreign_key "settings", "users" | add_foreign_key "settings", "users" | ||||
| add_foreign_key "tag_aliases", "tags" | add_foreign_key "tag_aliases", "tags" | ||||
| add_foreign_key "tag_implications", "tags" | |||||
| add_foreign_key "tag_implications", "tags", column: "parent_tag_id" | |||||
| add_foreign_key "tag_similarities", "tags" | add_foreign_key "tag_similarities", "tags" | ||||
| add_foreign_key "tag_similarities", "tags", column: "target_tag_id" | add_foreign_key "tag_similarities", "tags", column: "target_tag_id" | ||||
| add_foreign_key "user_ips", "ip_addresses" | add_foreign_key "user_ips", "ip_addresses" | ||||
| @@ -5,12 +5,30 @@ namespace :nico do | |||||
| require 'open-uri' | require 'open-uri' | ||||
| require 'nokogiri' | require 'nokogiri' | ||||
| fetch_thumbnail = -> url { | |||||
| fetch_thumbnail = -> url do | |||||
| html = URI.open(url, read_timeout: 60, 'User-Agent' => 'Mozilla/5.0').read | html = URI.open(url, read_timeout: 60, 'User-Agent' => 'Mozilla/5.0').read | ||||
| doc = Nokogiri::HTML(html) | doc = Nokogiri::HTML(html) | ||||
| doc.at('meta[name="thumbnail"]')&.[]('content').presence | doc.at('meta[name="thumbnail"]')&.[]('content').presence | ||||
| } | |||||
| end | |||||
| def sync_post_tags! post, desired_tag_ids | |||||
| desired_ids = desired_tag_ids.compact.to_set | |||||
| current_ids = post.tags.pluck(:id).to_set | |||||
| to_add = desired_ids - current_ids | |||||
| to_remove = current_ids - desired_ids | |||||
| Tag.where(id: to_add.to_a).find_each do |tag| | |||||
| begin | |||||
| PostTag.create!(post:, tag:) | |||||
| rescue ActiveRecord::RecordNotUnique | |||||
| ; | |||||
| end | |||||
| end | |||||
| PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each(&:discard!) | |||||
| end | |||||
| mysql_user = ENV['MYSQL_USER'] | mysql_user = ENV['MYSQL_USER'] | ||||
| mysql_pass = ENV['MYSQL_PASS'] | mysql_pass = ENV['MYSQL_PASS'] | ||||
| @@ -19,43 +37,53 @@ namespace :nico do | |||||
| { 'MYSQL_USER' => mysql_user, 'MYSQL_PASS' => mysql_pass }, | { 'MYSQL_USER' => mysql_user, 'MYSQL_PASS' => mysql_pass }, | ||||
| 'python3', "#{ nizika_nico_path }/get_videos.py") | 'python3', "#{ nizika_nico_path }/get_videos.py") | ||||
| if status.success? | |||||
| data = JSON.parse(stdout) | |||||
| data.each do |datum| | |||||
| post = Post.where('url LIKE ?', '%nicovideo.jp%').find { |post| | |||||
| post.url =~ %r{#{ Regexp.escape(datum['code']) }(?!\d)} | |||||
| } | |||||
| unless post | |||||
| title = datum['title'] | |||||
| url = "https://www.nicovideo.jp/watch/#{ datum['code'] }" | |||||
| thumbnail_base = fetch_thumbnail.(url) || '' rescue '' | |||||
| post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil) | |||||
| if thumbnail_base.present? | |||||
| post.thumbnail.attach( | |||||
| io: URI.open(thumbnail_base), | |||||
| filename: File.basename(URI.parse(thumbnail_base).path), | |||||
| content_type: 'image/jpeg') | |||||
| end | |||||
| post.save! | |||||
| post.resized_thumbnail! | |||||
| abort unless status.success? | |||||
| data = JSON.parse(stdout) | |||||
| data.each do |datum| | |||||
| post = Post.where('url LIKE ?', '%nicovideo.jp%').find { |post| | |||||
| post.url =~ %r{#{ Regexp.escape(datum['code']) }(?!\d)} | |||||
| } | |||||
| unless post | |||||
| title = datum['title'] | |||||
| url = "https://www.nicovideo.jp/watch/#{ datum['code'] }" | |||||
| thumbnail_base = fetch_thumbnail.(url) || '' rescue '' | |||||
| post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil) | |||||
| if thumbnail_base.present? | |||||
| post.thumbnail.attach( | |||||
| io: URI.open(thumbnail_base), | |||||
| filename: File.basename(URI.parse(thumbnail_base).path), | |||||
| content_type: 'image/jpeg') | |||||
| end | end | ||||
| post.save! | |||||
| post.resized_thumbnail! | |||||
| end | |||||
| kept_tags = post.tags.reload | |||||
| kept_non_nico_ids = kept_tags.where.not(category: 'nico').pluck(:id).to_set | |||||
| current_tags = post.tags.where(category: 'nico').pluck(:name).sort | |||||
| new_tags = datum['tags'].map { |tag| "nico:#{ tag }" }.sort | |||||
| if current_tags != new_tags | |||||
| post.tags.destroy(post.tags.where(name: current_tags)) | |||||
| tags_to_add = [] | |||||
| new_tags.each do |name| | |||||
| tag = Tag.find_or_initialize_by(name:) do |t| | |||||
| t.category = 'nico' | |||||
| end | |||||
| tags_to_add.concat([tag] + tag.linked_tags) | |||||
| end | |||||
| tags_to_add << Tag.tagme if post.tags.size < 20 | |||||
| tags_to_add << Tag.bot | |||||
| post.tags = (post.tags + tags_to_add).uniq | |||||
| desired_nico_ids = [] | |||||
| datum['tags'].each do |raw| | |||||
| name = "nico:#{ raw }" | |||||
| tag = Tag.find_or_initialize_by(name:) do |t| | |||||
| t.category = 'nico' | |||||
| end | end | ||||
| tag.save! if tag.new_record? | |||||
| desired_nico_ids << tag.id | |||||
| desired_nico_ids.concat(tag.linked_tags.pluck(:id)) | |||||
| end | end | ||||
| desired_nico_ids.uniq! | |||||
| desired_extra_ids = [] | |||||
| desired_extra_ids << Tag.tagme.id if kept_tags.size < 10 | |||||
| desired_extra_ids << Tag.bot.id | |||||
| desired_extra_ids.compact! | |||||
| desired_extra_ids.uniq! | |||||
| desired_all_ids = kept_non_nico_ids.to_a + desired_nico_ids + desired_extra_ids | |||||
| desired_all_ids.uniq! | |||||
| sync_post_tags!(post, desired_all_ids) | |||||
| end | end | ||||
| end | end | ||||
| end | end | ||||