多言語化の基盤作り (#55) (#421)

Reviewed-on: #421
Co-authored-by: miteruzo <miteruzo@naver.com>
このコミットはプルリクエスト #421 でマージされました。
このコミットが含まれているのは:
2026-10-03 02:00:09 +09:00
committed by みてるぞ
コミット e6ab941774
91個のファイルの変更、3370行の追加、1223行の削除
+1 -1
ファイルの表示
@@ -31,7 +31,7 @@ class MaterialVersionRecorder < VersionRecorder
{ url: @record.url,
parent: @record.parent,
tag: @record.tag,
tag_name: @record.tag&.name,
tag_name: @record.tag&.name('ja'),
tag_category: @record.tag&.category,
source_kind: @record.source_kind,
source_uri: @record.source_uri,
+5 -5
ファイルの表示
@@ -6,7 +6,7 @@ class PostBulkCreator
@host = host
end
def run
def run locale
results = Array.new(@posts.length)
mutex = Mutex.new
next_index = 0
@@ -27,7 +27,7 @@ class PostBulkCreator
break if index >= @posts.length
attributes = @posts[index]
results[index] = create_row(actor, attributes, index)
results[index] = create_row(locale, actor, attributes, index)
rescue StandardError => e
Rails.logger.error(
"post_bulk_creator_worker_failure #{ { error: e.class.name,
@@ -61,12 +61,12 @@ class PostBulkCreator
private
def create_row actor, attributes, index
def create_row locale, actor, attributes, index
preflight =
PostCreatePreflight.new(
attributes: attributes,
thumbnail: thumbnail_for(index, attributes),
host: @host).run
host: @host).run(locale)
if preflight[:existing_post_id].present?
return {
status: 'skipped',
@@ -76,7 +76,7 @@ class PostBulkCreator
post = PostCreator.new(
actor: actor,
attributes: normalised_attributes(attributes, preflight, index)).create!
attributes: normalised_attributes(attributes, preflight, index)).create!(locale)
result = {
status: 'created',
post: { id: post.id } }
+52 -41
ファイルの表示
@@ -8,13 +8,13 @@ class PostCreatePlan
@existing_tags_by_name = nil
end
def build!
direct_tag_specs, tag_sections = parse_direct_tag_specs
default_tag_specs = build_default_tag_specs(direct_tag_specs)
def build! locale
direct_tag_specs, tag_sections = parse_direct_tag_specs(locale)
default_tag_specs = build_default_tag_specs(locale, direct_tag_specs)
snapshot_tag_specs = merge_tag_specs(direct_tag_specs + default_tag_specs)
preload_existing_tags_by_name!(snapshot_tag_specs.map { _1[:name] })
validate_new_tag_specs!(snapshot_tag_specs)
post_tag_specs = expand_parent_tag_specs(snapshot_tag_specs)
preload_existing_tags_by_name!(locale, snapshot_tag_specs.map { _1[:name] })
validate_new_tag_specs!(locale, snapshot_tag_specs)
post_tag_specs = expand_parent_tag_specs(locale, snapshot_tag_specs)
video_ms = normalise_video_ms(snapshot_tag_specs)
validate_video_sections!(video_ms, tag_sections)
parent_post_ids = normalise_parent_post_ids
@@ -42,14 +42,16 @@ class PostCreatePlan
def tag_names = @attributes[:tags].to_s.split
def parse_direct_tag_specs
def parse_direct_tag_specs locale
tag_sections = { }
direct_tag_specs = []
tag_names.each do |raw_name|
tag_name, category, sections = parse_raw_tag_name(raw_name)
existing_tag = existing_tags_by_name[tag_name]
raise Tag::DeprecatedTagNormalisationError, [existing_tag.name] if existing_tag&.deprecated?
tag_name, category, sections = parse_raw_tag_name(locale, raw_name)
existing_tag = existing_tags_by_name(locale)[tag_name]
if existing_tag&.deprecated?
raise Tag::DeprecatedTagNormalisationError, [existing_tag.name(locale.language_code)]
end
direct_tag_specs << {
name: tag_name,
@@ -65,7 +67,7 @@ class PostCreatePlan
[merge_tag_specs(direct_tag_specs), tag_sections]
end
def parse_raw_tag_name raw_name
def parse_raw_tag_name locale, raw_name
name = raw_name.to_s
prefix, category =
Tag::CATEGORY_PREFIXES.find {
@@ -74,7 +76,7 @@ class PostCreatePlan
name = name.sub(/\A#{ prefix }/i, '')
sections = []
while (match = name.match(/\A(\S*?)\[([^\[\]\s]*)-([^\[\]\s]*)\](\S*)\z/))
while match = name.match(/\A(\S*?)\[([^\[\]\s]*)-([^\[\]\s]*)\](\S*)\z/)
name = "#{ match[1] }#{ match[4] }"
next if match[2].empty? && match[3].empty?
@@ -87,17 +89,17 @@ class PostCreatePlan
raise Tag::SectionLiteralParseError.new(raw_name, raw_name)
end
[resolved_tag_name(name), category&.to_sym, sections]
[resolved_tag_name(locale, name), category&.to_sym, sections]
end
def build_default_tag_specs direct_tag_specs
def build_default_tag_specs locale, direct_tag_specs
default_tag_specs = []
if direct_tag_specs.length < 10 && direct_tag_specs.none? { _1[:name] == TAGME_TAG_NAME }
default_tag_specs << {
name: TAGME_TAG_NAME,
category: :meta }
end
if direct_tag_specs.none? { deerjikist_tag_spec?(_1) }
if direct_tag_specs.none? { deerjikist_tag_spec?(locale, _1) }
default_tag_specs << {
name: NO_DEERJIKIST_TAG_NAME,
category: :meta }
@@ -106,17 +108,18 @@ class PostCreatePlan
default_tag_specs
end
def validate_new_tag_specs! specs
def validate_new_tag_specs! locale, specs
Array(specs).each do |spec|
next if existing_tags_by_name.key?(spec[:name])
next if existing_tags_by_name(locale).key?(spec[:name])
validate_new_tag_spec!(spec)
validate_new_tag_spec!(locale, spec)
end
end
def validate_new_tag_spec! spec
tag_name = TagName.new(name: spec[:name])
tag = Tag.new(category: spec[:category], tag_name:)
def validate_new_tag_spec! locale, spec
tag_name = TagName.new(name: spec[:name], language_code: locale.language_code,
script_code: locale.script_code, primary_flg: true)
tag = Tag.new(category: spec[:category])
return if tag_name.valid? && tag.valid?
post = Post.new
@@ -131,13 +134,15 @@ class PostCreatePlan
raise ActiveRecord::RecordInvalid, post
end
def expand_parent_tag_specs snapshot_tag_specs
existing_snapshot_tags = snapshot_tag_specs.filter_map { existing_tags_by_name[_1[:name]] }
def expand_parent_tag_specs locale, snapshot_tag_specs
existing_snapshot_tags = snapshot_tag_specs.filter_map do
existing_tags_by_name(locale)[_1[:name]]
end
expanded_parent_specs =
Tag.expand_parent_tags(existing_snapshot_tags)
.reject(&:deprecated?)
.map { |tag|
{ name: tag.name,
{ name: tag.name(locale.language_code),
category: tag.category.to_sym }
}
merge_tag_specs(snapshot_tag_specs + expanded_parent_specs)
@@ -156,33 +161,37 @@ class PostCreatePlan
}.values.sort_by { _1[:name] }
end
def existing_tags_by_name
def existing_tags_by_name locale
@existing_tags_by_name ||= begin
names = tag_names.map { canonical_tag_name_without_sections(_1) }.uniq
Tag.joins(:tag_name).where(tag_names: { name: names }).index_by(&:name)
names = tag_names.map { canonical_tag_name_without_sections(locale, _1) }.uniq
TagName.where(language_code: locale.language_code, primary_flg: true, name: names)
.includes(:tag)
.each_with_object({ }) { |tag_name, tags| tags[tag_name.name] = tag_name.tag }
end
end
def preload_existing_tags_by_name! names
def preload_existing_tags_by_name! locale, names
wanted_names = Array(names).map { _1.to_s }.reject(&:blank?).uniq
missing_names = wanted_names - existing_tags_by_name.keys
missing_names = wanted_names - existing_tags_by_name(locale).keys
return if missing_names.empty?
existing_tags_by_name.merge!(
Tag.joins(:tag_name)
.where(tag_names: { name: missing_names })
.index_by(&:name))
existing_tags_by_name(locale).merge!(
TagName.where(language_code: locale.language_code,
primary_flg: true,
name: missing_names)
.includes(:tag)
.each_with_object({ }) { |tag_name, tags| tags[tag_name.name] = tag_name.tag })
end
def canonical_tag_name_without_sections raw_name
name, = parse_raw_tag_name(raw_name)
def canonical_tag_name_without_sections locale, raw_name
name, = parse_raw_tag_name(locale, raw_name)
name
end
def deerjikist_tag_spec? spec
def deerjikist_tag_spec? locale, spec
return true if spec[:category] == :deerjikist
existing_tags_by_name[spec[:name]]&.deerjikist?
existing_tags_by_name(locale)[spec[:name]]&.deerjikist?
end
def normalise_parent_post_ids
@@ -260,10 +269,12 @@ class PostCreatePlan
end
end
def resolved_tag_name name
tag_name = TagName.includes(:canonical).find_by(name:)
return name if tag_name.nil?
def resolved_tag_name locale, name
tag_name = TagName.find_by(language_code: locale.language_code, name:)
return name if !(tag_name) || tag_name.primary?
(tag_name.canonical || tag_name).name
TagName.find_by(language_code: locale.language_code,
tag_id: tag_name.tag_id,
primary_flg: true)&.name || name
end
end
+3 -2
ファイルの表示
@@ -15,9 +15,10 @@ class PostCreatePreflight
@host = host
end
def run
def run locale
preview = PostImportPreviewer.new.preview_rows(
rows: [preview_row],
locale:,
fetch_metadata: false).first
if preview[:existing_post_id].present?
return {
@@ -53,7 +54,7 @@ class PostCreatePreflight
original_created_from: preview[:attributes]['original_created_from'],
original_created_before: preview[:attributes]['original_created_before'],
duration: preview[:attributes]['duration'],
video_ms: preview[:attributes]['video_ms'] }).build!
video_ms: preview[:attributes]['video_ms'] }).build!(locale)
{
url: plan[:url],
+36 -29
ファイルの表示
@@ -9,7 +9,7 @@ class PostCreator
@field_warnings = { }
end
def create!
def create! locale
thumbnail_attachment = prepare_thumbnail_attachment
post = Post.new(title: @attributes[:title].presence,
url: @attributes[:url],
@@ -21,14 +21,17 @@ class PostCreator
ApplicationRecord.transaction do
post.save!
post.thumbnail.attach(thumbnail_attachment) if thumbnail_attachment.present?
snapshot_tags = planned_snapshot_tags
post_tags = planned_post_tags
sections = planned_sections
TagVersioning.record_tag_snapshots!(snapshot_tags, created_by_user: @actor)
post.video_ms = planned_video_ms
snapshot_tags = planned_snapshot_tags(locale)
post_tags = planned_post_tags(locale)
sections = planned_sections(locale)
TagVersioning.record_tag_snapshots!(
snapshot_tags,
created_by_user: @actor,
language_code: locale.language_code)
post.video_ms = planned_video_ms(locale)
post.save!
sync_post_tags!(post, post_tags, sections)
sync_parent_posts!(post, planned_parent_post_ids)
sync_parent_posts!(post, planned_parent_post_ids(locale))
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor)
end
post
@@ -45,69 +48,73 @@ class PostCreator
thumbnail_base: @attributes[:thumbnail_base].presence)
end
def planned_snapshot_tags = planned_create_attributes[:snapshot_tags]
def planned_snapshot_tags(locale) = planned_create_attributes(locale)[:snapshot_tags]
def planned_post_tags = planned_create_attributes[:post_tags]
def planned_post_tags(locale) = planned_create_attributes(locale)[:post_tags]
def planned_sections = planned_create_attributes[:tag_sections]
def planned_sections(locale) = planned_create_attributes(locale)[:tag_sections]
def planned_parent_post_ids = planned_create_attributes[:normalised_parent_post_ids]
def planned_video_ms
planned_create_attributes[:video_ms]
def planned_parent_post_ids locale
planned_create_attributes(locale)[:normalised_parent_post_ids]
end
def planned_create_attributes
def planned_video_ms locale
planned_create_attributes(locale)[:video_ms]
end
def planned_create_attributes locale
@planned_create_attributes ||= begin
if @attributes.key?(:snapshot_tag_specs)
snapshot_tags = materialise_tags(@attributes[:snapshot_tag_specs] || [])
post_tags = materialise_tags(@attributes[:post_tag_specs] || [])
snapshot_tags = materialise_tags(locale, @attributes[:snapshot_tag_specs] || [])
post_tags = materialise_tags(locale, @attributes[:post_tag_specs] || [])
{
snapshot_tags: snapshot_tags,
post_tags: post_tags,
tag_sections: materialise_sections(
@attributes[:tag_sections] || { },
snapshot_tags,
post_tags),
post_tags,
locale),
normalised_parent_post_ids: @attributes[:normalised_parent_post_ids] || [],
video_ms: @attributes[:video_ms] }
else
build_materialised_plan
build_materialised_plan(locale)
end
end
end
def build_materialised_plan
plan = PostCreatePlan.new(attributes: @attributes).build!
snapshot_tags = materialise_tags(plan[:snapshot_tag_specs] || [])
post_tags = materialise_tags(plan[:post_tag_specs] || [])
def build_materialised_plan locale
plan = PostCreatePlan.new(attributes: @attributes).build!(locale)
snapshot_tags = materialise_tags(locale, plan[:snapshot_tag_specs] || [])
post_tags = materialise_tags(locale, plan[:post_tag_specs] || [])
{
snapshot_tags: snapshot_tags,
post_tags: post_tags,
tag_sections: materialise_sections(
plan[:tag_sections] || { },
snapshot_tags,
post_tags),
post_tags,
locale),
normalised_parent_post_ids: plan[:normalised_parent_post_ids] || [],
video_ms: plan[:video_ms] }
end
def materialise_tags specs
def materialise_tags locale, specs
Array(specs).each_with_object({ }) do |spec, tags|
name = spec[:name] || spec['name']
category = spec[:category] || spec['category']
next if name.blank? || category.blank?
tag = Tag.find_or_create_by_tag_name!(name, category:)
tag = Tag.find_or_create_by_tag_name!(locale, name, category:)
tag.update!(category:) if tag.category.to_sym != category.to_sym
tags[name] ||= tag
end.values
end
def materialise_sections sections_by_name, snapshot_tags, post_tags
tags_by_name = post_tags.index_by(&:name)
def materialise_sections sections_by_name, snapshot_tags, post_tags, locale = Locale.nipponese
tags_by_name = post_tags.index_by { _1.name(locale.language_code) }
snapshot_tags.each do |tag|
tags_by_name[tag.name] ||= tag
tags_by_name[tag.name(locale.language_code)] ||= tag
end
sections_by_name.each_with_object({ }) do |(tag_name, ranges), sections|
+20 -16
ファイルの表示
@@ -16,7 +16,7 @@ class PostImportPreviewer
THUMBNAIL_FETCH_WARNING = 'サムネールを取得できませんでした.'.freeze
METADATA_FETCH_WARNING = '自動取得に失敗しました.'.freeze
def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
def preview_rows rows:, locale: Locale.nipponese, fetch_metadata: true, metadata_cache: { }
prepared_rows = rows.map { prepare_row(_1) }
url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally
existing_posts =
@@ -30,13 +30,14 @@ class PostImportPreviewer
url_counts)
known_tags =
preload_known_tags(
locale,
prepared_rows,
fetch_metadata,
metadata_cache,
existing_posts,
url_counts)
prepared_rows.map { |row|
preview_row(row,
preview_row(locale, row,
fetch_metadata:,
metadata_cache:,
existing_posts:,
@@ -59,7 +60,7 @@ class PostImportPreviewer
source.merge(url_text: url, normal_url:, url_error: validate_url_safety(normal_url))
end
def preview_row row,
def preview_row locale, row,
fetch_metadata:,
metadata_cache:,
existing_posts:,
@@ -124,7 +125,8 @@ class PostImportPreviewer
attributes['url'] = url
validate_basic_data(attributes, validation_errors)
validate_preview_tags(merged_tags(tag_sources, provenance['tags']),
validate_preview_tags(locale,
merged_tags(tag_sources, provenance['tags']),
validation_errors,
known_tags)
validate_parents(attributes['parent_post_ids'], validation_errors, existing_parent_ids)
@@ -346,7 +348,7 @@ class PostImportPreviewer
{ data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] }, validation_errors: { } }
end
def preload_known_tags prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts
def preload_known_tags locale, prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts
names = prepared_rows.flat_map { |row|
attributes = initial_attributes(row)
provenance = initial_provenance(row)
@@ -358,15 +360,13 @@ class PostImportPreviewer
metadata = metadata_for(row[:normal_url], metadata_cache)
apply_metadata!(attributes, provenance, tag_sources, metadata[:data])
end
preview_tag_names(merged_tags(tag_sources, provenance['tags']))
preview_tag_names(locale, merged_tags(tag_sources, provenance['tags']))
}.compact.uniq
return { } if names.empty?
Tag.joins(:tag_name)
.where(tag_names: { name: names })
.includes(:tag_name)
.to_a
.index_by(&:name)
TagName.where(language_code: locale.language_code, primary_flg: true, name: names)
.includes(:tag)
.each_with_object({ }) { |tag_name, tags| tags[tag_name.name] = tag_name.tag }
end
def preload_parent_ids prepared_rows
@@ -437,19 +437,21 @@ class PostImportPreviewer
sources['automatic'].to_s
end
def preview_tag_names raw
def preview_tag_names locale, raw
names = raw.to_s.split
return [] if names.empty?
if names.any? { _1.downcase.start_with?('nico:') }
return []
end
names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
names.map { |name|
TagName.canonicalise(locale, name.sub(/\[.*\]\z/, '')).first
}
rescue Tag::SectionLiteralParseError
[]
end
def validate_preview_tags raw, errors, known_tags
def validate_preview_tags locale, raw, errors, known_tags
names = raw.to_s.split
return if names.empty?
if names.any? { _1.downcase.start_with?('nico:') }
@@ -457,9 +459,11 @@ class PostImportPreviewer
return
end
parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
parsed = names.map { |name|
TagName.canonicalise(locale, name.sub(/\[.*\]\z/, '')).first
}
existing = parsed.filter_map { known_tags[_1] }
deprecated = existing.select(&:deprecated?).map(&:name)
deprecated = existing.select(&:deprecated?).map { _1.name(locale.language_code) }
errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present?
rescue Tag::SectionLiteralParseError
errors[:tags] = ['タグ区間の記法が不正です.']
+3 -3
ファイルの表示
@@ -47,9 +47,9 @@ class PostMetadataFetcher
existing_tags =
Tag
.joins(:tag_name)
.where(tag_names: { name: names })
.index_by(&:name)
.joins(:tag_names)
.where(tag_names: { language_code: 'ja', primary_flg: true, name: names })
.index_by { _1.name('ja') }
names.map { |name|
tag = existing_tags[name]
+22 -3
ファイルの表示
@@ -1,5 +1,5 @@
class TagVersionRecorder < VersionRecorder
def self.record! tag:, event_type:, created_by_user:
def self.record! tag:, event_type:, created_by_user:, **_options
new(tag:, event_type:, created_by_user:).record!
end
@@ -14,10 +14,29 @@ class TagVersionRecorder < VersionRecorder
def record_key = :tag
def snapshot_attributes
{ name: @record.name,
names = snapshot_names
compatibility_names = names.fetch('ja')
{ name: compatibility_names.fetch('name'),
category: @record.category,
deprecated_at: @record.deprecated_at,
aliases: @record.snapshot_aliases.join(' '),
aliases: compatibility_names.fetch('aliases').map { _1.fetch('name') }.join(' '),
names:,
parent_tag_ids: @record.snapshot_parent_tag_ids.join(' ') }
end
def snapshot_names
@record.tag_names.order(:language_code, :name).group_by(&:language_code)
.transform_values do |tag_names|
primary = tag_names.find(&:primary?)
aliases = tag_names.reject(&:primary?).sort_by(&:name).map { |tag_name|
{ 'name' => tag_name.name, 'script_code' => tag_name.script_code }
}
{ 'name' => primary.name,
'script_code' => primary.script_code,
'auto_generated' => primary.auto_generated,
'aliases' => aliases }
end
end
end
+8 -8
ファイルの表示
@@ -1,22 +1,22 @@
class TagVersioning
def self.record! tag, event_type:, created_by_user:
TagVersionRecorder.record!(tag:, event_type:, created_by_user:)
def self.record! tag, event_type:, created_by_user:, language_code: 'ja'
TagVersionRecorder.record!(tag:, event_type:, created_by_user:, language_code:)
end
def self.ensure_snapshot! tag, created_by_user:
def self.ensure_snapshot! tag, created_by_user:, language_code: 'ja'
return if tag.tag_versions.exists?
TagVersionRecorder.record!(tag:, event_type: :create, created_by_user:)
TagVersionRecorder.record!(tag:, event_type: :create, created_by_user:, language_code:)
end
def self.record_tag_snapshot! tag, created_by_user:
def self.record_tag_snapshot! tag, created_by_user:, language_code: 'ja'
event_type = tag.tag_versions.exists? ? :update : :create
record!(tag, event_type:, created_by_user:)
record!(tag, event_type:, created_by_user:, language_code:)
end
def self.record_tag_snapshots! tags, created_by_user:
def self.record_tag_snapshots! tags, created_by_user:, language_code: 'ja'
tags.each do |tag|
record_tag_snapshot!(tag, created_by_user:)
record_tag_snapshot!(tag, created_by_user:, language_code:)
end
end
end
+3 -3
ファイルの表示
@@ -41,7 +41,7 @@ class TheatrePostSelector
def weighted_candidates
@weighted_candidates ||= begin
penalties = tag_penalties
posts = eligible_posts.includes(tags: :tag_name).to_a
posts = eligible_posts.includes(tags: :tag_names).to_a
posts.map do |post|
post_tags = post.tags.to_a
@@ -82,7 +82,7 @@ class TheatrePostSelector
def tag_penalty_json
return [] if tag_penalties.empty?
tags = Tag.where(id: tag_penalties.keys).includes(:tag_name).index_by(&:id)
tags = Tag.where(id: tag_penalties.keys).includes(:tag_names).index_by(&:id)
tag_penalties
.map { |tag_id, penalty|
@@ -113,7 +113,7 @@ class TheatrePostSelector
def light_tag_json tag
{ id: tag.id,
name: tag.name,
name: tag.name('ja'),
category: tag.category }
end
end