WIP: preserve failed Codex job issue-414-4ki4y15_

このコミットが含まれているのは:
Codex Night Worker
2026-11-19 12:51:14 +09:00
コミット 7de8b52811
33個のファイルの変更、517行の追加、279行の削除
+7
ファイルの表示
@@ -2,6 +2,13 @@ require 'set'
class MaterialsController < ApplicationController
around_action only: [:create, :update, :destroy] do |controller, action|
event_type = controller.action_name == 'destroy' ? 'discard' : controller.action_name
VersionRecorder.with_event(
event_type: "materials.#{ event_type }",
created_by_user: controller.current_user, &action)
end
rescue_from MaterialZipExporter::EmptyExportError, with: :render_zip_empty
rescue_from MaterialZipExporter::DuplicatePathError, with: :render_zip_duplicate_path
rescue_from MaterialZipExporter::MissingFileError, with: :render_zip_missing_file
+6
ファイルの表示
@@ -1,4 +1,10 @@
class NicoTagsController < ApplicationController
around_action only: [:update] do |controller, action|
VersionRecorder.with_event(
event_type: 'external_tags.update',
created_by_user: controller.current_user, &action)
end
def index
locale = resolve_locale!
+6
ファイルの表示
@@ -1,4 +1,10 @@
class PostsController < ApplicationController
around_action only: [:update] do |controller, action|
VersionRecorder.with_event(
event_type: 'posts.update',
created_by_user: controller.current_user, &action)
end
Event = Struct.new(:post, :tag, :user, :change_type, :timestamp, keyword_init: true)
MAX_BULK_REQUEST_BYTES = 40 * 1024 * 1024
+6
ファイルの表示
@@ -1,4 +1,10 @@
class TagChildrenController < ApplicationController
around_action only: [:create, :destroy] do |controller, action|
VersionRecorder.with_event(
event_type: 'tags.update',
created_by_user: controller.current_user, &action)
end
def create
return head :unauthorized unless current_user
return head :forbidden unless current_user.admin?
+6
ファイルの表示
@@ -4,6 +4,12 @@ require 'set'
class TagsController < ApplicationController
around_action only: [:update, :update_all] do |controller, action|
VersionRecorder.with_event(
event_type: 'tags.update',
created_by_user: controller.current_user, &action)
end
def index
locale = resolve_locale!
+17 -14
ファイルの表示
@@ -246,23 +246,26 @@ class WikiPagesController < ApplicationController
base_revision_id = params[:base_revision_id].presence
ApplicationRecord.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
tag.lock!
old_title = tag_name.name
if title != old_title
TagVersioning.ensure_snapshot!(tag, created_by_user: current_user)
end
VersionRecorder.with_event(
event_type: 'tag_descriptions.update', created_by_user: current_user) do
ApplicationRecord.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
tag.lock!
old_title = tag_name.name
if title != old_title
TagVersioning.ensure_snapshot!(tag, created_by_user: current_user)
end
tag_name.update!(name: title) if title != old_title
tag_name.update!(name: title) if title != old_title
message = params[:message].presence
TagDescriptionOperation.update!(locale, tag:, body:, updated_by_user: current_user,
message:, base_revision_id:)
message = params[:message].presence
TagDescriptionOperation.update!(locale, tag:, body:, updated_by_user: current_user,
message:, base_revision_id:)
if title != old_title
tag.reload
TagVersionRecorder.record!(tag:, event_type: :update, created_by_user: current_user)
if title != old_title
tag.reload
TagVersionRecorder.record!(tag:, event_type: :update, created_by_user: current_user)
end
end
end
+1
ファイルの表示
@@ -1,6 +1,7 @@
class Event < ApplicationRecord
has_many :post_versions
has_many :tag_versions
has_many :nico_tag_versions
has_many :material_versions
has_many :tag_description_versions
has_many :tag_group_versions
+33 -31
ファイルの表示
@@ -290,47 +290,49 @@ class Tag < ApplicationRecord
affected_post_ids = Set.new
Tag.transaction do
TagVersioning.ensure_snapshot!(target_tag, created_by_user:)
VersionRecorder.with_event(event_type: 'tags.update', created_by_user:) do
Tag.transaction do
TagVersioning.ensure_snapshot!(target_tag, created_by_user:)
Array(source_tags).compact.uniq.each do |source_tag|
source_tag => Tag
Array(source_tags).compact.uniq.each do |source_tag|
source_tag => Tag
next if source_tag == target_tag
next if source_tag == target_tag
TagVersioning.ensure_snapshot!(source_tag, created_by_user:)
TagVersioning.ensure_snapshot!(source_tag, created_by_user:)
source_tag.post_tags.find_each do |source_pt|
post_id = source_pt.post_id
affected_post_ids << post_id
source_pt.destroy!
unless PostTag.exists?(post_id:, tag: target_tag)
PostTag.create!(post_id:, tag: target_tag)
source_tag.post_tags.find_each do |source_pt|
post_id = source_pt.post_id
affected_post_ids << post_id
source_pt.destroy!
unless PostTag.exists?(post_id:, tag: target_tag)
PostTag.create!(post_id:, tag: target_tag)
end
end
if source_tag.tag_descriptions.exists?
source_tag.errors.add(:base, :invalid)
raise ActiveRecord::RecordInvalid, source_tag
end
TagVersioning.record!(source_tag, event_type: :discard, created_by_user:)
target_languages = target_tag.tag_names.where(primary_flg: true).pluck(:language_code)
source_tag.tag_names.where(language_code: target_languages)
.update_all(primary_flg: false, auto_generated: false, updated_at: Time.current)
source_tag.tag_names.update_all(tag_id: target_tag.id, updated_at: Time.current)
source_tag.destroy!
TagVersioning.record!(target_tag, event_type: :update, created_by_user:)
end
if source_tag.tag_descriptions.exists?
source_tag.errors.add(:base, :invalid)
raise ActiveRecord::RecordInvalid, source_tag
Post.where(id: affected_post_ids.to_a).find_each do |post|
PostVersionRecorder.ensure_snapshot!(post, created_by_user:)
PostVersionRecorder.record!(post:, event_type: :update, created_by_user:)
end
TagVersioning.record!(source_tag, event_type: :discard, created_by_user:)
target_languages = target_tag.tag_names.where(primary_flg: true).pluck(:language_code)
source_tag.tag_names.where(language_code: target_languages)
.update_all(primary_flg: false, auto_generated: false, updated_at: Time.current)
source_tag.tag_names.update_all(tag_id: target_tag.id, updated_at: Time.current)
source_tag.destroy!
TagVersioning.record!(target_tag, event_type: :update, created_by_user:)
# 投稿件数を再集計
target_tag.update_columns(post_count: PostTag.where(tag: target_tag).count)
end
Post.where(id: affected_post_ids.to_a).find_each do |post|
PostVersionRecorder.ensure_snapshot!(post, created_by_user:)
PostVersionRecorder.record!(post:, event_type: :update, created_by_user:)
end
# 投稿件数を再集計
target_tag.update_columns(post_count: PostTag.where(tag: target_tag).count)
end
target_tag.reload
+27 -25
ファイルの表示
@@ -15,37 +15,39 @@ class TagNameSanitisationRule < ApplicationRecord
end
def apply!
Language.find_each do |language|
TagName.where(language:).find_each do |candidate|
TagName.transaction do
tn = TagName.find_by(id: candidate.id)
next unless tn
VersionRecorder.with_event(event_type: 'tags.update', created_by_user: nil) do
Language.find_each do |language|
TagName.where(language:).find_each do |candidate|
TagName.transaction do
tn = TagName.find_by(id: candidate.id)
next unless tn
name = sanitise(tn.name)
next if name == tn.name
name = sanitise(tn.name)
next if name == tn.name
existing_tn = TagName.find_by(language:, name:)
source_tag = tn.tag
target_tag = existing_tn ? existing_tn.tag : source_tag
[source_tag, target_tag].uniq(&:id).sort_by(&:id).each(&:lock!)
TagVersioning.ensure_snapshot!(source_tag, created_by_user: nil)
existing_tn = TagName.find_by(language:, name:)
source_tag = tn.tag
target_tag = existing_tn ? existing_tn.tag : source_tag
[source_tag, target_tag].uniq(&:id).sort_by(&:id).each(&:lock!)
TagVersioning.ensure_snapshot!(source_tag, created_by_user: nil)
if existing_tn && target_tag != source_tag
Tag.merge_tags!(target_tag, source_tag)
tn.reload.destroy!
elsif existing_tn
if tn.primary?
existing_tn.destroy!
tn.update_columns(name:, updated_at: Time.current)
if existing_tn && target_tag != source_tag
Tag.merge_tags!(target_tag, source_tag)
tn.reload.destroy!
elsif existing_tn
if tn.primary?
existing_tn.destroy!
tn.update_columns(name:, updated_at: Time.current)
else
tn.destroy!
end
else
tn.destroy!
# TagName 側の自動サニタイズを回避
tn.update_columns(name:, updated_at: Time.current)
end
else
# TagName 側の自動サニタイズを回避
tn.update_columns(name:, updated_at: Time.current)
end
TagVersioning.record!(target_tag, event_type: :update, created_by_user: nil)
TagVersioning.record!(target_tag, event_type: :update, created_by_user: nil)
end
end
end
end
+20 -16
ファイルの表示
@@ -51,19 +51,23 @@ class MaterialSyncImporter
suppression: @import_block)
end
Material.transaction do
if material.persisted?
MaterialVersionRecorder.ensure_snapshot!(material,
created_by_user: @attributes[:updated_by_user])
VersionRecorder.with_event(
event_type: "materials.#{ event_type }",
created_by_user: @attributes[:updated_by_user]) do
Material.transaction do
if material.persisted?
MaterialVersionRecorder.ensure_snapshot!(material,
created_by_user: @attributes[:updated_by_user])
end
material.assign_attributes(material_attributes_for(material).merge(source))
material.discarded_at = nil if material.respond_to?(:discarded_at=)
material.file.attach(uploaded_blob) if uploaded_blob
material.save!
upsert_export_item!(material)
MaterialVersionRecorder.record!(material:,
event_type:,
created_by_user: @attributes[:updated_by_user])
end
material.assign_attributes(material_attributes_for(material).merge(source))
material.discarded_at = nil if material.respond_to?(:discarded_at=)
material.file.attach(uploaded_blob) if uploaded_blob
material.save!
upsert_export_item!(material)
MaterialVersionRecorder.record!(material:,
event_type:,
created_by_user: @attributes[:updated_by_user])
end
MaterialThumbnailGenerator.generate!(material)
@@ -200,10 +204,10 @@ class MaterialSyncImporter
end
def same_source_attributes? material, source
material.source_kind == source[:source_kind] \
&& material.source_uri == source[:source_uri] \
&& material.source_path == source[:source_path] \
&& material.source_file_id == source[:source_file_id]
material.source_kind == source[:source_kind] &&
material.source_uri == source[:source_uri] &&
material.source_path == source[:source_path] &&
material.source_file_id == source[:source_file_id]
end
def same_export_path? material
+17 -13
ファイルの表示
@@ -3,14 +3,16 @@ class MaterialSyncRunner
:errors, keyword_init: true)
def self.sync_enabled!
results = MaterialSyncSource.enabled.order(:id).map { |source| new(source).sync! }
VersionRecorder.with_event(event_type: 'materials.sync', created_by_user: nil) do
results = MaterialSyncSource.enabled.order(:id).map { |source| new(source).sync! }
Result.new(imported: results.sum(&:imported),
updated: results.sum(&:updated),
unchanged: results.sum(&:unchanged),
suppressed: results.sum(&:suppressed),
failed: results.sum(&:failed),
errors: results.flat_map(&:errors))
Result.new(imported: results.sum(&:imported),
updated: results.sum(&:updated),
unchanged: results.sum(&:unchanged),
suppressed: results.sum(&:suppressed),
failed: results.sum(&:failed),
errors: results.flat_map(&:errors))
end
end
def initialize source
@@ -21,13 +23,15 @@ class MaterialSyncRunner
result = Result.new(imported: 0, updated: 0, unchanged: 0,
suppressed: 0, failed: 0, errors: [])
if @source.source_kind == 'google_drive_path'
sync_google_drive_path!(result)
else
candidates.each do |candidate|
next if candidate.blank?
VersionRecorder.with_event(event_type: 'materials.sync', created_by_user: nil) do
if @source.source_kind == 'google_drive_path'
sync_google_drive_path!(result)
else
candidates.each do |candidate|
next if candidate.blank?
sync_candidate!(candidate, result)
sync_candidate!(candidate, result)
end
end
end
+7 -4
ファイルの表示
@@ -11,10 +11,13 @@ class MaterialSyncSuppressionRegistrar
def create!
suppression = nil
MaterialSyncSuppression.transaction do
suppression = MaterialSyncSuppression.create!(
@attributes.merge(created_by_user: @created_by_user))
discard_existing_materials!(suppression)
VersionRecorder.with_event(
event_type: 'materials.discard', created_by_user: @created_by_user) do
MaterialSyncSuppression.transaction do
suppression = MaterialSyncSuppression.create!(
@attributes.merge(created_by_user: @created_by_user))
discard_existing_materials!(suppression)
end
end
suppression
+3 -6
ファイルの表示
@@ -1,11 +1,11 @@
class MaterialVersionRecorder < VersionRecorder
EVENT_TYPES = ['create', 'update', 'discard', 'restore'].freeze
def self.record! material:, event_type:, event:, created_by_user:, file_snapshot: nil
def self.record! material:, event_type:, created_by_user:, event: nil, file_snapshot: nil
new(material:, event_type:, event:, created_by_user:, file_snapshot:).record!
end
def initialize material:, event_type:, event:, created_by_user:, file_snapshot: nil
def initialize material:, event_type:, created_by_user:, event: nil, file_snapshot: nil
@file_snapshot = file_snapshot
super(record: material, event_type:, event:, created_by_user:)
@@ -14,10 +14,7 @@ class MaterialVersionRecorder < VersionRecorder
def self.ensure_snapshot! material, created_by_user:
return if material.material_versions.exists?
event = Event.create!(event_type: 'materials.create',
created_by_user:,
created_at: material.created_at)
record!(material:, event_type: :create, event:,
record!(material:, event_type: :create,
created_by_user: material.created_by_user || created_by_user)
end
+4 -4
ファイルの表示
@@ -1,11 +1,11 @@
class NicoTagVersionRecorder < VersionRecorder
EVENT_TYPES = ['create', 'update'].freeze
def self.record! external_tag:, event_type:, created_by_user:
new(external_tag:, event_type:, created_by_user:).record!
def self.record! external_tag:, event_type:, created_by_user:, event: nil
new(external_tag:, event_type:, created_by_user:, event:).record!
end
def initialize external_tag:, event_type:, created_by_user:
super(record: external_tag, event_type:, created_by_user:)
def initialize external_tag:, event_type:, created_by_user:, event: nil
super(record: external_tag, event_type:, created_by_user:, event:)
end
private
+13 -3
ファイルの表示
@@ -7,6 +7,8 @@ class PostBulkCreator
end
def run locale
@event_context = nil
@event_mutex = Mutex.new
results = Array.new(@posts.length)
mutex = Mutex.new
next_index = 0
@@ -74,9 +76,17 @@ class PostBulkCreator
existing_post: preflight[:existing_post] }
end
post = PostCreator.new(
actor: actor,
attributes: normalised_attributes(attributes, preflight, index)).create!(locale)
# Keep preflight parallel, but publish the shared event with its first
# successful post before another worker can reference it.
post = @event_mutex.synchronize do
@event_context ||= { event_type: 'posts.create', created_by_user: actor }
VersionRecorder.with_event(
event_type: 'posts.create', created_by_user: actor, context: @event_context) do
PostCreator.new(
actor: actor,
attributes: normalised_attributes(attributes, preflight, index)).create!(locale)
end
end
result = {
status: 'created',
post: { id: post.id } }
+17 -15
ファイルの表示
@@ -18,21 +18,23 @@ class PostCreator
original_created_from: @attributes[:original_created_from].presence,
original_created_before: @attributes[:original_created_before].presence)
ApplicationRecord.transaction do
post.save!
post.thumbnail.attach(thumbnail_attachment) if thumbnail_attachment.present?
snapshot_tags = planned_snapshot_tags(locale)
post_tags = planned_post_tags(locale)
sections = planned_sections(locale)
TagVersioning.record_tag_snapshots!(
snapshot_tags,
created_by_user: @actor,
language_code: locale.language_code)
post.video_ms = planned_video_ms(locale)
post.save!
sync_post_tags!(post, post_tags, sections)
sync_parent_posts!(post, planned_parent_post_ids(locale))
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor)
VersionRecorder.with_event(event_type: 'posts.create', created_by_user: @actor) do
ApplicationRecord.transaction do
post.save!
post.thumbnail.attach(thumbnail_attachment) if thumbnail_attachment.present?
snapshot_tags = planned_snapshot_tags(locale)
post_tags = planned_post_tags(locale)
sections = planned_sections(locale)
TagVersioning.record_tag_snapshots!(
snapshot_tags,
created_by_user: @actor,
language_code: locale.language_code)
post.video_ms = planned_video_ms(locale)
post.save!
sync_post_tags!(post, post_tags, sections)
sync_parent_posts!(post, planned_parent_post_ids(locale))
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor)
end
end
post
rescue StandardError
+3 -6
ファイルの表示
@@ -1,19 +1,16 @@
class PostVersionRecorder < VersionRecorder
def self.record! post:, event_type:, event:, created_by_user:
def self.record! post:, event_type:, created_by_user:, event: nil
new(post:, event_type:, event:, created_by_user:).record!
end
def initialize post:, event_type:, event:, created_by_user:
def initialize post:, event_type:, created_by_user:, event: nil
super(record: post, event_type:, event:, created_by_user:)
end
def self.ensure_snapshot! post, created_by_user:
return if post.post_versions.exists?
event = Event.create!(event_type: 'posts.create',
created_by_user:,
created_at: post.created_at)
record!(post:, event_type: :create, event:, created_by_user:)
record!(post:, event_type: :create, created_by_user:)
end
private
+5 -1
ファイルの表示
@@ -42,9 +42,13 @@ class TagDescriptionOperation
version_no = creating ? 1 : description.version_no + 1
description.created_by_user_id = user.id if creating
description.update!(body:, version_no:, updated_by_user_id: user.id)
event_type = creating ? 'create' : 'update'
event = VersionRecorder.record_event!(
event_type: "tag_descriptions.#{ event_type }", created_by_user: user)
TagDescriptionVersion.create!(
event:,
tag_id: description.tag_id, language_code: description.language_code,
version_no:, event_type: creating ? 'create' : 'update', body:,
version_no:, event_type:, body:,
source_language_code: description.source_language_code,
source_version_no: description.source_version_no,
created_by_user_id: user.id, created_at: Time.current)
+3 -1
ファイルの表示
@@ -307,7 +307,9 @@ class TagGroupOperation
private_class_method :persist_placements!
def self.persist_version!(group:, attributes:, actor:, event_type:)
TagGroupVersion.create!(tag_group: group, version_no: group.version_no,
event = VersionRecorder.record_event!(
event_type: "tag_groups.#{ event_type }", created_by_user: actor)
TagGroupVersion.create!(tag_group: group, version_no: group.version_no, event:,
event_type:, name: group.name,
description: group.description,
members: attributes.fetch(:members),
+23 -21
ファイルの表示
@@ -1,29 +1,31 @@
class TagNameCompletion
class << self
def for_locale! locale
completed = Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
certified_names =
if locale.certified_tag_names_path.nil?
{ }
else
CertifiedTagNameTranslations.load!(locale.certified_tag_names_path, locale:)
end
Tag.where(id: certified_names.keys).lock.find_each(order: :asc) do |tag|
for_tag!(tag, locales: [locale],
certified_names: { locale.language_code => certified_names.fetch(tag.id) },
created_by_user: nil)
VersionRecorder.with_event(event_type: 'locales.create', created_by_user: nil) do
completed = Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
certified_names =
if locale.certified_tag_names_path.nil?
{ }
else
CertifiedTagNameTranslations.load!(locale.certified_tag_names_path, locale:)
end
Tag.where(id: certified_names.keys).lock.find_each(order: :asc) do |tag|
for_tag!(tag, locales: [locale],
certified_names: { locale.language_code => certified_names.fetch(tag.id) },
created_by_user: nil)
end
named_tags =
TagName.where(language_code: locale.language_code, primary_flg: true)
.select(:tag_id)
Tag.where.not(id: named_tags).lock.find_each(order: :asc) do |tag|
for_tag!(tag, locales: [locale], created_by_user: nil)
end
true
end
named_tags =
TagName.where(language_code: locale.language_code, primary_flg: true)
.select(:tag_id)
Tag.where.not(id: named_tags).lock.find_each(order: :asc) do |tag|
for_tag!(tag, locales: [locale], created_by_user: nil)
unless completed
raise ActiveRecord::RecordNotSaved, 'Locale tag completion was rolled back'
end
true
end
unless completed
raise ActiveRecord::RecordNotSaved, 'Locale tag completion was rolled back'
end
end
+2 -2
ファイルの表示
@@ -1,12 +1,12 @@
class TagVersionRecorder < VersionRecorder
def self.record! tag:, event_type:, event:, created_by_user:, **_options
def self.record! tag:, event_type:, created_by_user:, event: nil, **_options
Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
new(tag:, event_type:, event:, created_by_user:).record!
end
end
def initialize tag:, event_type:, event:, created_by_user:
def initialize tag:, event_type:, created_by_user:, event: nil
super(record: tag, event_type:, event:, created_by_user:)
end
+24 -2
ファイルの表示
@@ -1,7 +1,26 @@
class VersionRecorder
EVENT_TYPES = ['create', 'update', 'discard', 'restore'].freeze
def initialize record:, event_type:, event:, created_by_user:
# The scope is lazy: unchanged snapshots do not create an operation log.
def self.with_event event_type:, created_by_user:, context: nil
previous = ActiveSupport::IsolatedExecutionState[:version_event]
ActiveSupport::IsolatedExecutionState[:version_event] =
context || previous || { event_type:, created_by_user: }
yield
ensure
ActiveSupport::IsolatedExecutionState[:version_event] = previous
end
def self.record_event! event_type:, created_by_user:
context = ActiveSupport::IsolatedExecutionState[:version_event]
return Event.create!(event_type:, created_by_user:) unless context
return context[:event] if context[:event]&.persisted?
# Active Record resets persisted? when the creating transaction rolls back.
context[:event] = Event.create!(context.slice(:event_type, :created_by_user))
end
def initialize record:, event_type:, created_by_user:, event: nil
@record = record
@event_type = event_type.to_s
@event = event
@@ -43,7 +62,9 @@ class VersionRecorder
def base_attributes latest
{ version_no: (latest&.version_no || 0) + 1,
event_type: @event_type,
event: @event,
event: @event || self.class.record_event!(
event_type: "#{ event_resource }.#{ @event_type }",
created_by_user: @created_by_user),
created_by_user: @created_by_user,
created_at: Time.current }
end
@@ -90,5 +111,6 @@ class VersionRecorder
def snapshot_attributes = raise NotImplementedError
def record_class = @record.class
def event_resource = record_class.table_name
def event_types = self.class::EVENT_TYPES
end
+5 -3
ファイルの表示
@@ -12,9 +12,11 @@ module Youtube
video_ids = discover_video_ids
return if video_ids.empty?
video_ids.each_slice(50) do |ids|
@client.videos(ids).fetch('items', []).each do |item|
sync_video!(VideoItem.new(item))
VersionRecorder.with_event(event_type: 'posts.sync', created_by_user: nil) do
video_ids.each_slice(50) do |ids|
@client.videos(ids).fetch('items', []).each do |item|
sync_video!(VideoItem.new(item))
end
end
end
end
+99 -97
ファイルの表示
@@ -67,129 +67,131 @@ namespace :nico do
end
data = JSON.parse(stdout)
data.each do |datum|
code = datum['code']
post =
Post
.where('url REGEXP ?', "nicovideo\\.jp/watch/#{ Regexp.escape(code) }([^0-9]|$)")
.first
VersionRecorder.with_event(event_type: 'posts.sync', created_by_user: nil) do
data.each do |datum|
code = datum['code']
post =
Post
.where('url REGEXP ?', "nicovideo\\.jp/watch/#{ Regexp.escape(code) }([^0-9]|$)")
.first
title = datum['title']
original_created_at = datum['uploaded_at'] &&
Time.strptime(datum['uploaded_at'], '%Y-%m-%d %H:%M:%S')
original_created_from = original_created_at&.change(sec: 0)
original_created_before = original_created_from&.+(1.minute)
title = datum['title']
original_created_at = datum['uploaded_at'] &&
Time.strptime(datum['uploaded_at'], '%Y-%m-%d %H:%M:%S')
original_created_from = original_created_at&.change(sec: 0)
original_created_before = original_created_from&.+(1.minute)
post_created = false
post_changed = false
post_created = false
post_changed = false
if post
attrs = { title:, original_created_from:, original_created_before: }
if post
attrs = { title:, original_created_from:, original_created_before: }
unless post.thumbnail.attached?
thumbnail_base = fetch_thumbnail.(post.url) rescue nil
unless post.thumbnail.attached?
thumbnail_base = fetch_thumbnail.(post.url) rescue nil
if thumbnail_base.present?
attrs[:thumbnail_base] = thumbnail_base
begin
post.attach_thumbnail_from_url!(thumbnail_base)
rescue Post::RemoteThumbnailFetchFailed => e
payload = {
post_id: post.id,
thumbnail_base:,
error: e.class.name,
message: e.message }
Rails.logger.info("nico_sync_thumbnail_fetch_failed #{ payload.to_json }")
end
end
end
post.assign_attributes(attrs)
post_changed = post.changed?
if post_changed
post.save!
end
else
post_created = true
url = "https://www.nicovideo.jp/watch/#{ code }"
thumbnail_base = fetch_thumbnail.(url) rescue nil
post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil,
original_created_from:, original_created_before:)
if thumbnail_base.present?
attrs[:thumbnail_base] = thumbnail_base
begin
post.attach_thumbnail_from_url!(thumbnail_base)
rescue Post::RemoteThumbnailFetchFailed => e
payload = {
post_id: post.id,
post_id: nil,
thumbnail_base:,
error: e.class.name,
message: e.message }
Rails.logger.info("nico_sync_thumbnail_fetch_failed #{ payload.to_json }")
end
end
post.save!
sync_post_tags!(post, [Tag.tagme.id, Tag.bot.id, Tag.niconico.id, Tag.video.id])
end
post.assign_attributes(attrs)
post_changed = post.changed?
if post_changed
post.save!
end
else
post_created = true
url = "https://www.nicovideo.jp/watch/#{ code }"
thumbnail_base = fetch_thumbnail.(url) rescue nil
post = Post.new(title:, url:, thumbnail_base:, uploaded_user: nil,
original_created_from:, original_created_before:)
if thumbnail_base.present?
begin
post.attach_thumbnail_from_url!(thumbnail_base)
rescue Post::RemoteThumbnailFetchFailed => e
payload = {
post_id: nil,
thumbnail_base:,
error: e.class.name,
message: e.message }
Rails.logger.info("nico_sync_thumbnail_fetch_failed #{ payload.to_json }")
# 既存のタグ Id. 集合
kept_tag_ids = post.tags.pluck(:id).to_set
# 既存の外部タグ Id. 集合
kept_external_tag_ids = post.external_tags.nico.pluck(:id).to_set
# 記載すべき外部タグ Id. のリスト
desired_external_tag_ids = []
# 記載すべき内部タグ Id. のリスト
desired_tag_ids = kept_tag_ids.to_a
datum['tags'].each do |raw|
name = TagNameSanitisationRule.sanitise("nico:#{ raw }").delete_prefix('nico:')
tag = ExternalTag.find_or_create_by!(platform: :nico, name:)
unless tag.nico_tag_versions.exists?
NicoTagVersionRecorder.record!(external_tag: tag,
event_type: :create,
created_by_user: nil)
end
desired_external_tag_ids << tag.id
# 新たに記載される外部タグと連携される内部タグを記載
# 連携タグは記載すれども消除せず.
unless tag.id.in?(kept_external_tag_ids)
desired_tag_ids.concat(tag.linked_tags.pluck(:id))
end
end
post.save!
sync_post_tags!(post, [Tag.tagme.id, Tag.bot.id, Tag.niconico.id, Tag.video.id])
end
# 既存のタグ Id. 集合
kept_tag_ids = post.tags.pluck(:id).to_set
# 既存の外部タグ Id. 集合
kept_external_tag_ids = post.external_tags.nico.pluck(:id).to_set
# 記載すべき外部タグ Id. のリスト
desired_external_tag_ids = []
# 記載すべき内部タグ Id. のリスト
desired_tag_ids = kept_tag_ids.to_a
datum['tags'].each do |raw|
name = TagNameSanitisationRule.sanitise("nico:#{ raw }").delete_prefix('nico:')
tag = ExternalTag.find_or_create_by!(platform: :nico, name:)
unless tag.nico_tag_versions.exists?
NicoTagVersionRecorder.record!(external_tag: tag,
event_type: :create,
created_by_user: nil)
deerjikist = Deerjikist.find_by(platform: :nico, code: datum['user'])
if deerjikist
desired_tag_ids << deerjikist.tag_id
elsif !(Tag.where(id: kept_tag_ids).where(category: :deerjikist).exists?)
desired_tag_ids << Tag.no_deerjikist.id
end
desired_external_tag_ids << tag.id
# 新たに記載される外部タグと連携される内部タグを記載
# 連携タグは記載すれども消除せず.
unless tag.id.in?(kept_external_tag_ids)
desired_tag_ids.concat(tag.linked_tags.pluck(:id))
end
end
deerjikist = Deerjikist.find_by(platform: :nico, code: datum['user'])
if deerjikist
desired_tag_ids << deerjikist.tag_id
elsif !(Tag.where(id: kept_tag_ids).where(category: :deerjikist).exists?)
desired_tag_ids << Tag.no_deerjikist.id
end
desired_external_tag_ids.uniq!
desired_tag_ids.uniq!
# 外部タグの記載に際しては “bot 操作” タグを記載しなぃ.
if kept_tag_ids != desired_tag_ids.to_set
desired_tag_ids << Tag.bot.id
desired_external_tag_ids.uniq!
desired_tag_ids.uniq!
end
tags_changed =
kept_tag_ids != desired_tag_ids.to_set ||
kept_external_tag_ids != desired_external_tag_ids.to_set
# 外部タグの記載に際しては “bot 操作” タグを記載しなぃ.
if kept_tag_ids != desired_tag_ids.to_set
desired_tag_ids << Tag.bot.id
desired_tag_ids.uniq!
end
sync_post_tags!(post, desired_tag_ids, current_tag_ids: kept_tag_ids)
sync_post_external_tags!(post, desired_external_tag_ids,
current_external_tag_ids: kept_external_tag_ids)
tags_changed =
kept_tag_ids != desired_tag_ids.to_set ||
kept_external_tag_ids != desired_external_tag_ids.to_set
if post_created
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: nil)
elsif post_changed || tags_changed
PostVersionRecorder.ensure_snapshot!(post, created_by_user: nil)
PostVersionRecorder.record!(post:, event_type: :update, created_by_user: nil)
sync_post_tags!(post, desired_tag_ids, current_tag_ids: kept_tag_ids)
sync_post_external_tags!(post, desired_external_tag_ids,
current_external_tag_ids: kept_external_tag_ids)
if post_created
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: nil)
elsif post_changed || tags_changed
PostVersionRecorder.ensure_snapshot!(post, created_by_user: nil)
PostVersionRecorder.record!(post:, event_type: :update, created_by_user: nil)
end
end
end
end
+2
ファイルの表示
@@ -92,6 +92,7 @@ RSpec.describe 'Counts artifact policy and external references' do
auto_generated: false, aliases: [] } })
external = create(:external_tag, name: 'public-external')
row = NicoTagVersion.create!(external_tag: external, version_no: 1, event_type: 'create',
event_id: create_event!(event_type: 'unknown').id,
linked_tag_ids: tag.id.to_s,
created_at: Time.utc(2024))
post = create(:post)
@@ -109,6 +110,7 @@ RSpec.describe 'Counts artifact policy and external references' do
expect(HistoryAccess.visible(NicoTagVersion.where(id: row.id))).to exist
end
public_row = NicoTagVersion.create!(external_tag: external, version_no: 2, event_type: 'update',
event_id: create_event!(event_type: 'unknown').id,
linked_tag_ids: '',
created_at: Time.utc(2024, 7))
HistoryAccess.set(user: nil) do
+14 -14
ファイルの表示
@@ -46,13 +46,13 @@ RSpec.describe VersionRecord, type: :model do
it 'makes NicoTagVersion read only after create' do
version = NicoTagVersion.create!(
external_tag: nico_tag,
version_no: 1,
event_type: 'create',
linked_tag_ids: '',
created_at: Time.current,
created_by_user: nil
)
event_id: create_event!(event_type: 'unknown').id,
external_tag: nico_tag,
version_no: 1,
event_type: 'create',
linked_tag_ids: '',
created_at: Time.current,
created_by_user: nil)
expect {
version.update!(linked_tag_ids: '123')
@@ -73,13 +73,13 @@ RSpec.describe VersionRecord, type: :model do
it 'prevents NicoTagVersion destroy' do
version = NicoTagVersion.create!(
external_tag: nico_tag,
version_no: 1,
event_type: 'create',
linked_tag_ids: '',
created_at: Time.current,
created_by_user: nil
)
event_id: create_event!(event_type: 'unknown').id,
external_tag: nico_tag,
version_no: 1,
event_type: 'create',
linked_tag_ids: '',
created_at: Time.current,
created_by_user: nil)
expect {
version.destroy!
+3
ファイルの表示
@@ -13,6 +13,7 @@ RSpec.describe 'Counts masked search', type: :request do
before do
[1, 2].each do |number|
TagVersion.create!(tag:, version_no: number,
event_id: create_event!(event_type: 'unknown').id,
event_type: number == 1 ? 'create' : 'discard', name: '秘密のタグ',
category: 'general', aliases: '秘密の別名', parent_tag_ids: '',
names: { ja: { name: '秘密のタグ', script_code: 'Jpan', auto_generated: false,
@@ -113,6 +114,7 @@ RSpec.describe 'Counts masked search', type: :request do
it 'searches public literal markers and treats axis predicates as explicit' do
public_tag = create(:tag, primary_name: '********')
TagVersion.create!(tag: public_tag, version_no: 1, event_type: 'create',
event_id: create_event!(event_type: 'unknown').id,
name: '********', category: 'general', aliases: '', parent_tag_ids: '',
names: { ja: { name: '********', script_code: 'Jpan', auto_generated: false,
aliases: [] } }, created_at: Time.utc(2023))
@@ -177,6 +179,7 @@ RSpec.describe 'Counts masked search', type: :request do
it 'rejects private names reached through a post tag snapshot' do
post = create(:post)
PostVersion.create!(post:, version_no: 1, event_type: 'create', title: 'post',
event_id: create_event!(event_type: 'unknown').id,
url: post.url, tags: '', parent_post_ids: '',
tags_json: [{ tag_id: tag.id, version_no: 1, name: '秘密のタグ',
category: 'general', sections: [] }],
+4
ファイルの表示
@@ -21,6 +21,7 @@ RSpec.describe 'Counts table API', type: :request do
names: { ja: { name: 'alpha', script_code: 'Jpan', auto_generated: false, aliases: [] } },
created_at: Time.utc(2023))
NicoTagVersion.create!(external_tag: external, version_no: 1,
event_id: create_event!(event_type: 'unknown').id,
event_type: 'create', linked_tag_ids: '', created_at: Time.utc(2023))
snapshots = [
{ tag_id: tag.id, version_no: 1, name: 'alpha', category: 'general', sections: [] },
@@ -61,9 +62,11 @@ RSpec.describe 'Counts table API', type: :request do
before do
NicoTagVersion.create!(external_tag: external, version_no: 2,
event_id: create_event!(event_type: 'unknown').id,
event_type: 'update', linked_tag_ids: tag.id.to_s,
created_at: Time.utc(2024, 1, 1))
NicoTagVersion.create!(external_tag: external, version_no: 3,
event_id: create_event!(event_type: 'unknown').id,
event_type: 'update', linked_tag_ids: '',
created_at: Time.utc(2024, 2, 1))
end
@@ -487,6 +490,7 @@ RSpec.describe 'Counts table API', type: :request do
it 'allows public identity predicates without exposing private snapshot fields' do
PostVersion.create!(post: record, version_no: 4, event_type: 'discard',
event_id: create_event!(event_type: 'unknown').id,
url: record.url, title: 'hidden title', tags: '', tags_json: [],
parent_post_ids: '', created_at: Time.utc(2024, 3))
+2 -1
ファイルの表示
@@ -184,7 +184,7 @@ RSpec.describe 'NicoTags', type: :request do
expect(linked_versions.map(&:created_by_user_id)).to all(eq(admin.id))
event = event_rows.last
expect(event).to include(
'event_type' => 'tags.update', 'created_by_user_id' => admin.id)
'event_type' => 'external_tags.update', 'created_by_user_id' => admin.id)
expect(linked_versions.pluck(:event_id).uniq).to eq([event.fetch('id')])
versions = nico_tag.reload.nico_tag_versions.order(:version_no)
@@ -192,6 +192,7 @@ RSpec.describe 'NicoTags', type: :request do
expect(versions.last.linked_tag_ids.split.map(&:to_i))
.to match_array([linked_a.id, linked_b.id])
expect(versions.last.created_by_user_id).to eq(admin.id)
expect(versions.last.event_id).to eq(event.fetch('id'))
end
it 'clears existing links and records the empty mapping for a member' do
+4
ファイルの表示
@@ -14,12 +14,14 @@ RSpec.describe 'Post history privacy', type: :request do
before do
['create', 'discard'].each_with_index do |event, index|
TagVersion.create!(tag:, version_no: index + 1, event_type: event,
event_id: create_event!(event_type: 'unknown').id,
name: 'private-history-tag', category: 'general', aliases: '', parent_tag_ids: '',
names: { ja: { name: 'private-history-tag', script_code: 'Jpan',
auto_generated: false, aliases: [] } },
created_at: origin + index.hours)
end
NicoTagVersion.create!(external_tag: external, version_no: 1, event_type: 'create',
event_id: create_event!(event_type: 'unknown').id,
linked_tag_ids: tag.id.to_s,
created_at: origin)
['create', 'discard', 'restore', 'update'].each_with_index do |event, index|
@@ -32,6 +34,7 @@ RSpec.describe 'Post history privacy', type: :request do
[]
end
PostVersion.create!(post: post_record, version_no: index + 1, event_type: event,
event_id: create_event!(event_type: 'unknown').id,
title: "version #{ index + 1 }", url: post_record.url, tags: '',
tags_json: snapshots, parent_post_ids: '', created_at: origin + (index + 1).days)
end
@@ -51,6 +54,7 @@ RSpec.describe 'Post history privacy', type: :request do
it 'keeps every version through a later discard private after another restore' do
['discard', 'restore'].each_with_index do |event, index|
PostVersion.create!(post: post_record, version_no: index + 5, event_type: event,
event_id: create_event!(event_type: 'unknown').id,
title: 'later snapshot', url: post_record.url, tags: '', tags_json: [],
parent_post_ids: '', created_at: origin + (index + 5).days)
end
+2
ファイルの表示
@@ -32,8 +32,10 @@ RSpec.describe 'Counts export contracts' do
before do
NicoTagVersion.create!(external_tag: external, version_no: 1,
event_id: create_event!(event_type: 'unknown').id,
event_type: 'create', linked_tag_ids: '', created_at: Time.utc(2023))
NicoTagVersion.create!(external_tag: external, version_no: 2,
event_id: create_event!(event_type: 'unknown').id,
event_type: 'update', linked_tag_ids: tag.id.to_s, created_at: Time.utc(2024))
end
+4
ファイルの表示
@@ -15,6 +15,8 @@ RSpec.describe NicoTagVersionRecorder do
external_tag:, version_no: 1, event_type: 'create',
name: 'nico:raw tag[]', linked_tag_ids: '', created_by_user: member)
expect(external_tag.reload.nico_tag_versions).to contain_exactly(version)
expect(version.event).to have_attributes(
event_type: 'external_tags.create', created_by_user: member)
end
it 'uses the latest history number when the record has no version_no column' do
@@ -25,6 +27,8 @@ RSpec.describe NicoTagVersionRecorder do
expect(second).to have_attributes(version_no: 2, name: 'nico:raw tag[]',
linked_tag_ids: tag.id.to_s)
expect(second.event).to have_attributes(
event_type: 'external_tags.update', created_by_user: member)
expect(first.reload.name).to eq('nico:raw tag[]')
expect(external_tag.reload.has_attribute?(:version_no)).to be(false)
end
+128
ファイルの表示
@@ -18,6 +18,9 @@ RSpec.describe VersionRecorder do
expect(version.version_no).to eq(1)
expect(post_record.reload.version_no).to eq(1)
expect(version.event).to have_attributes(
event_type: 'posts.create', created_by_user: member, reason: nil)
expect(version.event.created_at).to be_present
end
it 'updates record version_no when creating the next version' do
@@ -44,6 +47,7 @@ RSpec.describe VersionRecorder do
post: post_record,
event_type: :create,
created_by_user: member)
events = event_rows
expect {
version =
@@ -56,6 +60,130 @@ RSpec.describe VersionRecorder do
}.not_to change(PostVersion, :count)
expect(post_record.reload.version_no).to eq(1)
expect(event_rows).to eq(events)
end
it 'shares the outer operation between different history types' do
tag = create(:tag)
external = create(:external_tag)
material = Material.create!(url: 'https://example.com/event-material')
member
post_record
versions = []
expect {
described_class.with_event(event_type: 'posts.create', created_by_user: member) do
versions << PostVersionRecorder.record!(
post: post_record, event_type: :create, created_by_user: member)
described_class.with_event(event_type: 'tags.update', created_by_user: nil) do
versions << TagVersioning.record_tag_snapshot!(tag, created_by_user: member)
versions << NicoTagVersionRecorder.record!(
external_tag: external, event_type: :create, created_by_user: member)
versions << MaterialVersionRecorder.record!(
material:, event_type: :create, created_by_user: member)
end
end
}.to change(Event, :count).by(1)
expect(versions.map(&:event_id).uniq).to eq([versions.first.event_id])
expect(versions.first.event).to have_attributes(
event_type: 'posts.create', created_by_user: member)
expect(versions.first.event.nico_tag_versions).to contain_exactly(versions[2])
end
it 'recreates a rolled-back event before the next successful history' do
member
post_record
events = event_rows
version = nil
described_class.with_event(event_type: 'posts.sync', created_by_user: nil) do
ApplicationRecord.transaction(requires_new: true) do
PostVersionRecorder.record!(
post: post_record, event_type: :create, created_by_user: member)
raise ActiveRecord::Rollback
end
expect(event_rows).to eq(events)
expect(post_record.reload.post_versions).to be_empty
expect(post_record.version_no).to eq(1)
version = PostVersionRecorder.record!(
post: post_record, event_type: :create, created_by_user: member)
end
expect(Event.count).to eq(events.length + 1)
expect(version.event).to have_attributes(event_type: 'posts.sync', created_by_user: nil)
end
it 'clears the operation after an exception' do
expect {
described_class.with_event(event_type: 'posts.sync', created_by_user: nil) do
raise 'operation failed'
end
}.to raise_error('operation failed')
version = PostVersionRecorder.record!(
post: post_record, event_type: :create, created_by_user: member)
expect(version.event).to have_attributes(event_type: 'posts.create', created_by_user: member)
end
it 'does not create an event for an empty operation' do
expect {
described_class.with_event(event_type: 'posts.sync', created_by_user: nil) { nil }
}.not_to change(Event, :count)
end
it 'rolls back the event when its version fails validation' do
post_record.update_columns(url: '')
member
events = event_rows
expect {
ApplicationRecord.transaction(requires_new: true) do
PostVersionRecorder.record!(
post: post_record, event_type: :create, created_by_user: member)
end
}.to raise_error(ActiveRecord::RecordInvalid)
expect(post_record.reload.post_versions).to be_empty
expect(event_rows).to eq(events)
end
it 'keeps an explicitly supplied event' do
event = Event.create!(event_type: 'posts.create', created_by_user: member)
expect {
version = PostVersionRecorder.record!(
post: post_record, event_type: :create, created_by_user: member, event:)
expect(version.event).to eq(event)
}.not_to change(Event, :count)
end
it 'keeps simultaneous operation scopes separate' do
ready = Queue.new
continuation = Queue.new
events = ['posts.sync', 'materials.sync'].to_h do |event_type|
event = instance_double(Event, persisted?: true)
allow(Event).to receive(:create!).with({ event_type:, created_by_user: nil })
.and_return(event)
[event_type, event]
end
workers = events.keys.map do |event_type|
Thread.new do
described_class.with_event(event_type:, created_by_user: nil) do
ready << true
continuation.pop
described_class.record_event!(event_type: 'tags.create', created_by_user: nil)
end
end
end
2.times { ready.pop }
2.times { continuation << true }
expect(workers.map(&:value)).to eq(events.values)
ensure
workers&.each(&:join)
end
it 'raises when record version_no is older than the latest version' do