This commit is contained in:
@@ -63,3 +63,5 @@ gem 'diff-lcs'
|
|||||||
gem 'dotenv-rails'
|
gem 'dotenv-rails'
|
||||||
|
|
||||||
gem 'whenever', require: false
|
gem 'whenever', require: false
|
||||||
|
|
||||||
|
gem 'discard'
|
||||||
|
|||||||
@@ -90,6 +90,8 @@ GEM
|
|||||||
crass (1.0.6)
|
crass (1.0.6)
|
||||||
date (3.4.1)
|
date (3.4.1)
|
||||||
diff-lcs (1.6.2)
|
diff-lcs (1.6.2)
|
||||||
|
discard (1.4.0)
|
||||||
|
activerecord (>= 4.2, < 9.0)
|
||||||
dotenv (3.1.8)
|
dotenv (3.1.8)
|
||||||
dotenv-rails (3.1.8)
|
dotenv-rails (3.1.8)
|
||||||
dotenv (= 3.1.8)
|
dotenv (= 3.1.8)
|
||||||
@@ -420,6 +422,7 @@ DEPENDENCIES
|
|||||||
bootsnap
|
bootsnap
|
||||||
brakeman
|
brakeman
|
||||||
diff-lcs
|
diff-lcs
|
||||||
|
discard
|
||||||
dotenv-rails
|
dotenv-rails
|
||||||
gollum
|
gollum
|
||||||
image_processing (~> 1.14)
|
image_processing (~> 1.14)
|
||||||
|
|||||||
@@ -1,7 +1,3 @@
|
|||||||
require 'open-uri'
|
|
||||||
require 'nokogiri'
|
|
||||||
|
|
||||||
|
|
||||||
class PostsController < ApplicationController
|
class PostsController < ApplicationController
|
||||||
# GET /posts
|
# GET /posts
|
||||||
def index
|
def index
|
||||||
@@ -77,7 +73,7 @@ class PostsController < ApplicationController
|
|||||||
post.thumbnail.attach(thumbnail)
|
post.thumbnail.attach(thumbnail)
|
||||||
if post.save
|
if post.save
|
||||||
post.resized_thumbnail!
|
post.resized_thumbnail!
|
||||||
post.tags = Tag.normalise_tags(tag_names)
|
sync_post_tags!(post, Tag.normalise_tags(tag_names))
|
||||||
render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }),
|
render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }),
|
||||||
status: :created
|
status: :created
|
||||||
else
|
else
|
||||||
@@ -110,8 +106,10 @@ class PostsController < ApplicationController
|
|||||||
original_created_before = params[:original_created_before]
|
original_created_before = params[:original_created_before]
|
||||||
|
|
||||||
post = Post.find(params[:id].to_i)
|
post = Post.find(params[:id].to_i)
|
||||||
tags = post.tags.where(category: 'nico').to_a + Tag.normalise_tags(tag_names)
|
if post.update(title:, original_created_from:, original_created_before:)
|
||||||
if post.update(title:, tags:, original_created_from:, original_created_before:)
|
sync_post_tags!(post,
|
||||||
|
(post.tags.where(category: 'nico').to_a +
|
||||||
|
Tag.normalise_tags(tag_names)))
|
||||||
render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }),
|
render json: post.as_json(include: { tags: { only: [:id, :name, :category, :post_count] } }),
|
||||||
status: :ok
|
status: :ok
|
||||||
else
|
else
|
||||||
@@ -128,7 +126,11 @@ class PostsController < ApplicationController
|
|||||||
def filtered_posts
|
def filtered_posts
|
||||||
tag_names = params[:tags]&.split(' ')
|
tag_names = params[:tags]&.split(' ')
|
||||||
match_type = params[:match]
|
match_type = params[:match]
|
||||||
tag_names.present? ? filter_posts_by_tags(tag_names, match_type) : Post.all
|
if tag_names.present?
|
||||||
|
filter_posts_by_tags(tag_names, match_type)
|
||||||
|
else
|
||||||
|
Post.all
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def filter_posts_by_tags tag_names, match_type
|
def filter_posts_by_tags tag_names, match_type
|
||||||
@@ -142,4 +144,28 @@ class PostsController < ApplicationController
|
|||||||
end
|
end
|
||||||
posts.distinct
|
posts.distinct
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def sync_post_tags! post, desired_tags
|
||||||
|
desired_tags.each do |t|
|
||||||
|
t.save! if t.new_record?
|
||||||
|
end
|
||||||
|
|
||||||
|
desired_ids = desired_tags.map(&:id).to_set
|
||||||
|
current_ids = post.tags.pluck(:id).to_set
|
||||||
|
|
||||||
|
to_add = desired_ids - current_ids
|
||||||
|
to_remove = current_ids - desired_ids
|
||||||
|
|
||||||
|
Tag.where(id: to_add).find_each do |tag|
|
||||||
|
begin
|
||||||
|
PostTag.create!(post:, tag:, created_user: current_user)
|
||||||
|
rescue ActiveRecord::RecordNotUnique
|
||||||
|
;
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each do |pt|
|
||||||
|
pt.discard_by!(current_user)
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
require 'mini_magick'
|
|
||||||
|
|
||||||
|
|
||||||
class Post < ApplicationRecord
|
class Post < ApplicationRecord
|
||||||
|
require 'mini_magick'
|
||||||
|
|
||||||
belongs_to :parent, class_name: 'Post', optional: true, foreign_key: 'parent_id'
|
belongs_to :parent, class_name: 'Post', optional: true, foreign_key: 'parent_id'
|
||||||
belongs_to :uploaded_user, class_name: 'User', optional: true
|
belongs_to :uploaded_user, class_name: 'User', optional: true
|
||||||
has_many :post_tags, dependent: :destroy
|
|
||||||
has_many :tags, through: :post_tags
|
has_many :post_tags, dependent: :delete_all, inverse_of: :post
|
||||||
|
has_many :active_post_tags, -> { kept }, class_name: 'PostTag', inverse_of: :post
|
||||||
|
has_many :post_tags_with_discarded, -> { with_discarded }, class_name: 'PostTag'
|
||||||
|
has_many :tags, through: :active_post_tags
|
||||||
has_many :user_post_views, dependent: :destroy
|
has_many :user_post_views, dependent: :destroy
|
||||||
has_many :post_similarities_as_post,
|
has_many :post_similarities_as_post,
|
||||||
class_name: 'PostSimilarity',
|
class_name: 'PostSimilarity',
|
||||||
|
|||||||
@@ -1,7 +1,25 @@
|
|||||||
class PostTag < ApplicationRecord
|
class PostTag < ApplicationRecord
|
||||||
|
include Discard::Model
|
||||||
|
|
||||||
belongs_to :post
|
belongs_to :post
|
||||||
belongs_to :tag, counter_cache: :post_count
|
belongs_to :tag, counter_cache: :post_count
|
||||||
|
belongs_to :created_user, class_name: 'User', optional: true
|
||||||
|
belongs_to :deleted_user, class_name: 'User', optional: true
|
||||||
|
|
||||||
validates :post_id, presence: true
|
validates :post_id, presence: true
|
||||||
validates :tag_id, presence: true
|
validates :tag_id, presence: true
|
||||||
|
validates :post_id, uniqueness: {
|
||||||
|
scope: :tag_id,
|
||||||
|
conditions: -> { where(discarded_at: nil) } }
|
||||||
|
|
||||||
|
def discard_by! deleted_user
|
||||||
|
return self if discarded?
|
||||||
|
|
||||||
|
transaction do
|
||||||
|
update!(discarded_at: Time.current, deleted_user:)
|
||||||
|
Tag.where(id: tag_id).update_all('post_count = GREATEST(post_count - 1, 0)')
|
||||||
|
end
|
||||||
|
|
||||||
|
self
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
class Tag < ApplicationRecord
|
class Tag < ApplicationRecord
|
||||||
has_many :post_tags, dependent: :destroy
|
has_many :post_tags, dependent: :delete_all, inverse_of: :tag
|
||||||
has_many :posts, through: :post_tags
|
has_many :active_post_tags, -> { kept }, class_name: 'PostTag', :inverse_of: :tag
|
||||||
|
has_many :post_tags_with_discarded, -> { with_discarded }, class_name: 'PostTag'
|
||||||
|
has_many :posts, through: :active_post_tags
|
||||||
has_many :tag_aliases, dependent: :destroy
|
has_many :tag_aliases, dependent: :destroy
|
||||||
|
|
||||||
has_many :nico_tag_relations, foreign_key: :nico_tag_id, dependent: :destroy
|
has_many :nico_tag_relations, foreign_key: :nico_tag_id, dependent: :destroy
|
||||||
@@ -35,13 +37,13 @@ class Tag < ApplicationRecord
|
|||||||
'meta:' => 'meta' }.freeze
|
'meta:' => 'meta' }.freeze
|
||||||
|
|
||||||
def self.tagme
|
def self.tagme
|
||||||
@tagme ||= Tag.find_or_initialize_by(name: 'タグ希望') do |tag|
|
@tagme ||= Tag.find_or_create_by!(name: 'タグ希望') do |tag|
|
||||||
tag.category = 'meta'
|
tag.category = 'meta'
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.bot
|
def self.bot
|
||||||
@bot ||= Tag.find_or_initialize_by(name: 'bot操作') do |tag|
|
@bot ||= Tag.find_or_create_by!(name: 'bot操作') do |tag|
|
||||||
tag.category = 'meta'
|
tag.category = 'meta'
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
class AddDiscardToPostTags < ActiveRecord::Migration[8.0]
|
||||||
|
def change
|
||||||
|
add_column :post_tags, :discarded_at, :datetime
|
||||||
|
add_index :post_tags, :discarded_at
|
||||||
|
|
||||||
|
add_column :post_tags, :is_active, :boolean,
|
||||||
|
as: 'discarded_at IS NULL', stored: true
|
||||||
|
|
||||||
|
add_column :post_tags, :active_unique_key, :string,
|
||||||
|
as: "CASE WHEN discarded_at IS NULL THEN CONCAT(post_id, ':', tag_id) ELSE NULL END",
|
||||||
|
stored: true
|
||||||
|
|
||||||
|
add_index :post_tags, :active_unique_key, unique: true, name: 'idx_post_tags_active_unique'
|
||||||
|
|
||||||
|
add_index :post_tags, [:post_id, :discarded_at]
|
||||||
|
add_index :post_tags, [:tag_id, :discarded_at]
|
||||||
|
end
|
||||||
|
end
|
||||||
Generated
+19
-1
@@ -10,7 +10,7 @@
|
|||||||
#
|
#
|
||||||
# It's strongly recommended that you check this file into your version control system.
|
# It's strongly recommended that you check this file into your version control system.
|
||||||
|
|
||||||
ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do
|
ActiveRecord::Schema[8.0].define(version: 2025_10_11_200300) do
|
||||||
create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||||
t.string "name", null: false
|
t.string "name", null: false
|
||||||
t.string "record_type", null: false
|
t.string "record_type", null: false
|
||||||
@@ -70,9 +70,16 @@ ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do
|
|||||||
t.bigint "deleted_user_id"
|
t.bigint "deleted_user_id"
|
||||||
t.datetime "created_at", null: false
|
t.datetime "created_at", null: false
|
||||||
t.datetime "updated_at", null: false
|
t.datetime "updated_at", null: false
|
||||||
|
t.datetime "discarded_at"
|
||||||
|
t.virtual "is_active", type: :boolean, as: "(`discarded_at` is null)", stored: true
|
||||||
|
t.virtual "active_unique_key", type: :string, as: "(case when (`discarded_at` is null) then concat(`post_id`,_utf8mb4':',`tag_id`) else NULL end)", stored: true
|
||||||
|
t.index ["active_unique_key"], name: "idx_post_tags_active_unique", unique: true
|
||||||
t.index ["created_user_id"], name: "index_post_tags_on_created_user_id"
|
t.index ["created_user_id"], name: "index_post_tags_on_created_user_id"
|
||||||
t.index ["deleted_user_id"], name: "index_post_tags_on_deleted_user_id"
|
t.index ["deleted_user_id"], name: "index_post_tags_on_deleted_user_id"
|
||||||
|
t.index ["discarded_at"], name: "index_post_tags_on_discarded_at"
|
||||||
|
t.index ["post_id", "discarded_at"], name: "index_post_tags_on_post_id_and_discarded_at"
|
||||||
t.index ["post_id"], name: "index_post_tags_on_post_id"
|
t.index ["post_id"], name: "index_post_tags_on_post_id"
|
||||||
|
t.index ["tag_id", "discarded_at"], name: "index_post_tags_on_tag_id_and_discarded_at"
|
||||||
t.index ["tag_id"], name: "index_post_tags_on_tag_id"
|
t.index ["tag_id"], name: "index_post_tags_on_tag_id"
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -107,6 +114,15 @@ ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do
|
|||||||
t.index ["tag_id"], name: "index_tag_aliases_on_tag_id"
|
t.index ["tag_id"], name: "index_tag_aliases_on_tag_id"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
create_table "tag_implications", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||||
|
t.bigint "tag_id", null: false
|
||||||
|
t.bigint "parent_tag_id", null: false
|
||||||
|
t.datetime "created_at", null: false
|
||||||
|
t.datetime "updated_at", null: false
|
||||||
|
t.index ["parent_tag_id"], name: "index_tag_implications_on_parent_tag_id"
|
||||||
|
t.index ["tag_id"], name: "index_tag_implications_on_tag_id"
|
||||||
|
end
|
||||||
|
|
||||||
create_table "tag_similarities", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
create_table "tag_similarities", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||||
t.bigint "tag_id", null: false
|
t.bigint "tag_id", null: false
|
||||||
t.bigint "target_tag_id", null: false
|
t.bigint "target_tag_id", null: false
|
||||||
@@ -176,6 +192,8 @@ ActiveRecord::Schema[8.0].define(version: 2025_09_09_075500) do
|
|||||||
add_foreign_key "posts", "users", column: "uploaded_user_id"
|
add_foreign_key "posts", "users", column: "uploaded_user_id"
|
||||||
add_foreign_key "settings", "users"
|
add_foreign_key "settings", "users"
|
||||||
add_foreign_key "tag_aliases", "tags"
|
add_foreign_key "tag_aliases", "tags"
|
||||||
|
add_foreign_key "tag_implications", "tags"
|
||||||
|
add_foreign_key "tag_implications", "tags", column: "parent_tag_id"
|
||||||
add_foreign_key "tag_similarities", "tags"
|
add_foreign_key "tag_similarities", "tags"
|
||||||
add_foreign_key "tag_similarities", "tags", column: "target_tag_id"
|
add_foreign_key "tag_similarities", "tags", column: "target_tag_id"
|
||||||
add_foreign_key "user_ips", "ip_addresses"
|
add_foreign_key "user_ips", "ip_addresses"
|
||||||
|
|||||||
@@ -5,12 +5,30 @@ namespace :nico do
|
|||||||
require 'open-uri'
|
require 'open-uri'
|
||||||
require 'nokogiri'
|
require 'nokogiri'
|
||||||
|
|
||||||
fetch_thumbnail = -> url {
|
fetch_thumbnail = -> url do
|
||||||
html = URI.open(url, read_timeout: 60, 'User-Agent' => 'Mozilla/5.0').read
|
html = URI.open(url, read_timeout: 60, 'User-Agent' => 'Mozilla/5.0').read
|
||||||
doc = Nokogiri::HTML(html)
|
doc = Nokogiri::HTML(html)
|
||||||
|
|
||||||
doc.at('meta[name="thumbnail"]')&.[]('content').presence
|
doc.at('meta[name="thumbnail"]')&.[]('content').presence
|
||||||
}
|
end
|
||||||
|
|
||||||
|
def sync_post_tags! post, desired_tag_ids
|
||||||
|
desired_ids = desired_tag_ids.compact.to_set
|
||||||
|
current_ids = post.tags.pluck(:id).to_set
|
||||||
|
|
||||||
|
to_add = desired_ids - current_ids
|
||||||
|
to_remove = current_ids - desired_ids
|
||||||
|
|
||||||
|
Tag.where(id: to_add.to_a).find_each do |tag|
|
||||||
|
begin
|
||||||
|
PostTag.create!(post:, tag:)
|
||||||
|
rescue ActiveRecord::RecordNotUnique
|
||||||
|
;
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
PostTag.where(post_id: post.id, tag_id: to_remove.to_a).kept.find_each(&:discard!)
|
||||||
|
end
|
||||||
|
|
||||||
mysql_user = ENV['MYSQL_USER']
|
mysql_user = ENV['MYSQL_USER']
|
||||||
mysql_pass = ENV['MYSQL_PASS']
|
mysql_pass = ENV['MYSQL_PASS']
|
||||||
@@ -19,43 +37,53 @@ namespace :nico do
|
|||||||
{ 'MYSQL_USER' => mysql_user, 'MYSQL_PASS' => mysql_pass },
|
{ 'MYSQL_USER' => mysql_user, 'MYSQL_PASS' => mysql_pass },
|
||||||
'python3', "#{ nizika_nico_path }/get_videos.py")
|
'python3', "#{ nizika_nico_path }/get_videos.py")
|
||||||
|
|
||||||
if status.success?
|
abort unless status.success?
|
||||||
data = JSON.parse(stdout)
|
|
||||||
data.each do |datum|
|
|
||||||
post = Post.where('url LIKE ?', '%nicovideo.jp%').find { |post|
|
|
||||||
post.url =~ %r{#{ Regexp.escape(datum['code']) }(?!\d)}
|
|
||||||
}
|
|
||||||
unless post
|
|
||||||
title = datum['title']
|
|
||||||
url = "https://www.nicovideo.jp/watch/#{ datum['code'] }"
|
|
||||||
thumbnail_base = fetch_thumbnail.(url) || '' rescue ''
|
|
||||||
post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil)
|
|
||||||
if thumbnail_base.present?
|
|
||||||
post.thumbnail.attach(
|
|
||||||
io: URI.open(thumbnail_base),
|
|
||||||
filename: File.basename(URI.parse(thumbnail_base).path),
|
|
||||||
content_type: 'image/jpeg')
|
|
||||||
end
|
|
||||||
post.save!
|
|
||||||
post.resized_thumbnail!
|
|
||||||
end
|
|
||||||
|
|
||||||
current_tags = post.tags.where(category: 'nico').pluck(:name).sort
|
data = JSON.parse(stdout)
|
||||||
new_tags = datum['tags'].map { |tag| "nico:#{ tag }" }.sort
|
data.each do |datum|
|
||||||
if current_tags != new_tags
|
post = Post.where('url LIKE ?', '%nicovideo.jp%').find { |post|
|
||||||
post.tags.destroy(post.tags.where(name: current_tags))
|
post.url =~ %r{#{ Regexp.escape(datum['code']) }(?!\d)}
|
||||||
tags_to_add = []
|
}
|
||||||
new_tags.each do |name|
|
unless post
|
||||||
tag = Tag.find_or_initialize_by(name:) do |t|
|
title = datum['title']
|
||||||
t.category = 'nico'
|
url = "https://www.nicovideo.jp/watch/#{ datum['code'] }"
|
||||||
end
|
thumbnail_base = fetch_thumbnail.(url) || '' rescue ''
|
||||||
tags_to_add.concat([tag] + tag.linked_tags)
|
post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil)
|
||||||
end
|
if thumbnail_base.present?
|
||||||
tags_to_add << Tag.tagme if post.tags.size < 20
|
post.thumbnail.attach(
|
||||||
tags_to_add << Tag.bot
|
io: URI.open(thumbnail_base),
|
||||||
post.tags = (post.tags + tags_to_add).uniq
|
filename: File.basename(URI.parse(thumbnail_base).path),
|
||||||
|
content_type: 'image/jpeg')
|
||||||
end
|
end
|
||||||
|
post.save!
|
||||||
|
post.resized_thumbnail!
|
||||||
end
|
end
|
||||||
|
|
||||||
|
kept_tags = post.tags.reload
|
||||||
|
kept_non_nico_ids = kept_tags.where.not(category: 'nico').pluck(:id).to_set
|
||||||
|
|
||||||
|
desired_nico_ids = []
|
||||||
|
datum['tags'].each do |raw|
|
||||||
|
name = "nico:#{ raw }"
|
||||||
|
tag = Tag.find_or_initialize_by(name:) do |t|
|
||||||
|
t.category = 'nico'
|
||||||
|
end
|
||||||
|
tag.save! if tag.new_record?
|
||||||
|
desired_nico_ids << tag.id
|
||||||
|
desired_nico_ids.concat(tag.linked_tags.pluck(:id))
|
||||||
|
end
|
||||||
|
desired_nico_ids.uniq!
|
||||||
|
|
||||||
|
desired_extra_ids = []
|
||||||
|
desired_extra_ids << Tag.tagme.id if kept_tags.size < 10
|
||||||
|
desired_extra_ids << Tag.bot.id
|
||||||
|
desired_extra_ids.compact!
|
||||||
|
desired_extra_ids.uniq!
|
||||||
|
|
||||||
|
desired_all_ids = kept_non_nico_ids.to_a + desired_nico_ids + desired_extra_ids
|
||||||
|
desired_all_ids.uniq!
|
||||||
|
|
||||||
|
sync_post_tags!(post, desired_all_ids)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
Reference in New Issue
Block a user