アイヌ語追加 (#428) (#433)

Reviewed-on: #433
Co-authored-by: miteruzo <miteruzo@naver.com>
このコミットはプルリクエスト #433 でマージされました。
このコミットが含まれているのは:
2026-10-20 20:44:14 +09:00
committed by みてるぞ
コミット 084a090cab
294個のファイルの変更、13978行の追加、3461行の削除
+105
ファイルの表示
@@ -0,0 +1,105 @@
require 'json'
require 'pathname'
class CertifiedTagNameTranslations
FORMAT = 'btrc-hub.certified-tag-name-translations.v1'.freeze
class Error < ArgumentError
;
end
def self.load! path, locale:
new(path, locale:).load!
end
def initialize path, locale:
@path = path
@locale = locale
end
def load!
document = read_document
unless document.is_a?(Hash) && document['format'] == FORMAT
fail!('invalid format or version')
end
source_code = document['source_locale']
target_code = document['target_locale']
unless target_code.is_a?(String) && target_code == Locale.codes.fetch(@locale.code)
fail!("target_locale #{ target_code.inspect } does not match the new Locale")
end
unless source_code.is_a?(String) && Locale.codes.value?(source_code)
fail!("invalid source_locale #{ source_code.inspect }")
end
source_locale = Locale.find_by(code: source_code)
fail!("source Locale #{ source_code.inspect } does not exist") unless source_locale
fail!('source_locale and target_locale must differ') if source_code == target_code
entries = validate_entries(document['entries'], source_code, target_code)
resolve_tags(entries, source_locale.language_code)
end
private
def read_document
unless (@path.is_a?(String) && !@path.include?("\0") &&
Pathname.new(@path).absolute?)
fail!('certified_tag_names_path must be an absolute file path')
end
fail!('JSON file does not exist or is not a regular file') unless File.file?(@path)
text = File.read(@path, encoding: Encoding::UTF_8)
fail!('JSON file is not valid UTF-8') unless text.valid_encoding?
JSON.parse(text)
rescue JSON::ParserError, SystemCallError, IOError, EncodingError => error
fail!("cannot read JSON: #{ error.message }")
end
def validate_entries entries, source_code, target_code
fail!('entries must be an array') unless entries.is_a?(Array)
sources = {}
targets = {}
entries.each_with_index.map do |entry, index|
fail!("entry #{ index + 1 } must be an object") unless entry.is_a?(Hash)
source = entry[source_code]
target = entry[target_code]
unless source.is_a?(String) && source.present? && target.is_a?(String) && target.present?
fail!("entry #{ index + 1 }: missing or empty #{ source_code }/#{ target_code } name")
end
fail!("duplicate source name #{ source.inspect }") if sources.key?(source)
fail!("duplicate target name #{ target.inspect }") if targets.key?(target)
sources[source] = true
targets[target] = true
[source, target]
end
end
def resolve_tags entries, source_language
source_names = entries.map(&:first)
# SQL may use a case/accent-insensitive collation; the Ruby keys are exact.
matches = TagName.where(language_code: source_language, primary_flg: true,
name: source_names).pluck(:name, :tag_id).group_by(&:first)
translations = {}
entries.each do |source, target|
rows = matches.fetch(source, [])
next if rows.empty?
unless rows.length == 1
fail!("source primary name #{ source.inspect } must match exactly one tag")
end
tag_id = rows.first.last
if translations.key?(tag_id)
fail!("multiple source primary names resolve to Tag##{ tag_id }: #{ source.inspect }")
end
translations[tag_id] = target
end
translations
end
def fail! message
raise Error, "Certified tag names #{@path.inspect}: #{ message }"
end
end
+1 -1
ファイルの表示
@@ -105,7 +105,7 @@ class PostCreator
category = spec[:category] || spec['category']
next if name.blank? || category.blank?
tag = Tag.find_or_create_by_tag_name!(locale, name, category:)
tag = Tag.find_or_create_by_tag_name!(locale, name, category:, created_by_user: @actor)
tag.update!(category:) if tag.category.to_sym != category.to_sym
tags[name] ||= tag
end.values
+76 -30
ファイルの表示
@@ -13,42 +13,26 @@ class SharedResourceIdAllocator
['tag_description_versions', 'tag_id']],
'wiki' => [['tag_group_versions', 'tag_group_id']] }.freeze
LOCK_TIMEOUT = 10
CONNECTION_LOCKS = ObjectSpace::WeakMap.new
class LockCleanup
def initialize &finish
@finish = finish
end
def trigger_transactional_callbacks? = false
def before_committed! = nil
def committed!(**) = @finish.call(self)
def rolledback!(**) = @finish.call(self)
end
private_constant :LockCleanup
def self.assign! record
space, tables = SPACES.find { |_name, names| names.include?(record.class.table_name) }
raise ArgumentError, 'Unknown shared ID resource' unless tables
connection = record.class.connection
transaction = record.class.current_transaction
unless transaction.open?
raise ActiveRecord::ActiveRecordError, 'Shared ID requires a transaction'
end
database = connection.pool.db_config.database
name = "btrc:ids:#{ Digest::SHA256.hexdigest(database)[0, 16] }:#{ space }"
quoted_name = connection.quote(name)
acquired = record.class.uncached do
connection.select_value("SELECT GET_LOCK(#{ quoted_name }, #{ LOCK_TIMEOUT })")
end
unless acquired.to_i == 1
raise ActiveRecord::LockWaitTimeout, "Shared ID lock unavailable: #{ space }"
end
release = lambda do
result = record.class.uncached do
connection.select_value("SELECT RELEASE_LOCK(#{ quoted_name })")
end
unless result.to_i == 1
raise ActiveRecord::ActiveRecordError, "Shared ID lock release failed: #{ space }"
end
end
begin
transaction.after_commit(&release)
transaction.after_rollback(&release)
rescue StandardError
release.call
raise
end
lock_space!(record.class, space)
# 明示-tic な Id. 指定は許可する.
return if record.id.present?
@@ -66,6 +50,68 @@ class SharedResourceIdAllocator
record.id = next_id
end
def self.lock_space! record_class, space
raise ArgumentError, 'Unknown shared ID resource' unless SPACES.key?(space)
connection = record_class.connection
transaction = connection.current_transaction
unless transaction.open?
raise ActiveRecord::ActiveRecordError, 'Shared ID requires a transaction'
end
database = connection.pool.db_config.database
name = "btrc:ids:#{ Digest::SHA256.hexdigest(database)[0, 16] }:#{ space }"
held_locks = CONNECTION_LOCKS[connection] ||= {}
return if held_locks.key?(name)
quoted_name = connection.quote(name)
acquired = record_class.uncached do
connection.select_value("SELECT GET_LOCK(#{ quoted_name }, #{ LOCK_TIMEOUT })")
end
unless acquired.to_i == 1
raise ActiveRecord::LockWaitTimeout, "Shared ID lock unavailable: #{ space }"
end
held_locks[name] = true
finish = lambda do |cleanup|
next unless held_locks.key?(name)
# Savepoint callbacks may run before the outer transaction ends.
# Keep one session lock and transfer its cleanup to the remaining parent.
parent = connection.current_transaction
if parent.open?
connection.add_transaction_record(cleanup)
next
end
result = record_class.uncached do
connection.select_value("SELECT RELEASE_LOCK(#{ quoted_name })")
end
unless result.to_i == 1
raise ActiveRecord::ActiveRecordError, "Shared ID lock release failed: #{ space }"
end
held_locks.delete(name)
rescue StandardError
# Never return a session with an untracked advisory lock to the pool.
begin
connection.disconnect!
ensure
held_locks.clear
end
raise
end
begin
# Rails finalises remaining records in ensure if another callback raises.
connection.add_transaction_record(LockCleanup.new(&finish))
rescue StandardError
begin
connection.disconnect!
ensure
held_locks.clear
end
raise
end
end
def self.next_ids connection, tables
expiry = connection.select_value('SELECT @@SESSION.information_schema_stats_expiry').to_i
begin
+2
ファイルの表示
@@ -5,6 +5,7 @@ class TagDescriptionOperation
def self.create! locale, tag:, body:, created_by_user:, message: nil
ApplicationRecord.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
tag.lock!
raise ActiveRecord::RecordNotUnique if TagDescription.exists?(
tag_id: tag.id, language_code: locale.language_code)
@@ -19,6 +20,7 @@ class TagDescriptionOperation
def self.update! locale, tag:, body:, updated_by_user:, message: nil, base_revision_id: nil
ApplicationRecord.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
tag.lock!
description = TagDescription.lock.find_by!(
tag_id: tag.id, language_code: locale.language_code)
+122
ファイルの表示
@@ -0,0 +1,122 @@
class TagNameCompletion
class << self
def for_locale! locale
completed = Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
certified_names =
if locale.certified_tag_names_path.nil?
{ }
else
CertifiedTagNameTranslations.load!(locale.certified_tag_names_path, locale:)
end
Tag.where(id: certified_names.keys).lock.find_each(order: :asc) do |tag|
for_tag!(tag, locales: [locale],
certified_names: { locale.language_code => certified_names.fetch(tag.id) },
created_by_user: nil)
end
named_tags =
TagName.where(language_code: locale.language_code, primary_flg: true)
.select(:tag_id)
Tag.where.not(id: named_tags).lock.find_each(order: :asc) do |tag|
for_tag!(tag, locales: [locale], created_by_user: nil)
end
true
end
unless completed
raise ActiveRecord::RecordNotSaved, 'Locale tag completion was rolled back'
end
end
def for_tag! tag, locales: nil, certified_names: { }, created_by_user: nil
locales ||=
Locale.where(deprecated_at: nil)
.order(:language_code, :created_at, :script_code).lock
Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
# Creation includes every language in its first snapshot.
creating = tag.new_record?
tag.save! if creating
completed = tag.with_lock do
primary_names = tag.tag_names.where(primary_flg: true).lock.index_by(&:language_code)
certified_names.each do |language, name|
primary = primary_names[language]
next unless primary
next if primary.name == name && !primary.auto_generated
raise CertifiedTagNameTranslations::Error,
"Tag##{ tag.id } already has a primary #{ language } name; " +
"cannot register certified name #{ name.inspect }"
end
missing_locales = locales.to_a.uniq(&:language_code).reject { |locale|
primary_names.key?(locale.language_code)
}
next true if missing_locales.empty? && !creating
TagVersioning.ensure_snapshot!(tag, created_by_user:) unless creating
missing_locales.each do |locale|
primary_names[locale.language_code] =
if certified_names.key?(locale.language_code)
create_certified_name!(locale, tag, certified_names.fetch(locale.language_code))
else
source_language = locale.language_code == 'ja' ? 'ain' : 'ja'
name = TagName.generate_name(locale, tag, primary_names[source_language])
create_name!(locale, tag, name)
end
end
tag.association(:tag_names).reset
event_type = creating ? :create : :update
version = TagVersioning.record!(tag, event_type:, created_by_user:)
unless version&.persisted?
raise ActiveRecord::RecordNotSaved, 'Tag version was not persisted'
end
true
end
unless completed
raise ActiveRecord::RecordNotSaved, 'Tag completion was rolled back'
end
end
end
private
def create_certified_name! locale, tag, name
persist_name!(locale, tag, name, auto_generated: false)
rescue ActiveRecord::RecordInvalid => error
raise unless error.record.is_a?(TagName)
raise CertifiedTagNameTranslations::Error,
"Tag##{ tag.id } certified name #{ name.inspect }: " +
error.record.errors.full_messages.join(', ')
rescue ActiveRecord::RecordNotUnique => error
raise CertifiedTagNameTranslations::Error,
"Tag##{ tag.id } certified name #{ name.inspect } conflicts: #{ error.message }"
end
def create_name! locale, tag, name
persist_name!(locale, tag, name, auto_generated: true)
rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => error
fallback = "Tag_##{ tag.id }"
invalid_name =
if error.is_a?(ActiveRecord::RecordNotUnique)
TagName.where(language_code: locale.language_code, name:).lock.exists?
else
error.record.errors[:name].present? &&
error.record.errors.attribute_names.all? { _1 == :name }
end
raise if name == fallback || !(invalid_name)
persist_name!(locale, tag, fallback, auto_generated: true)
end
def persist_name! locale, tag, name, auto_generated:
tag_name = TagName.transaction(requires_new: true) do
TagName.create!(tag:, name:, language_code: locale.language_code,
script_code: locale.script_code, primary_flg: true, auto_generated:)
end
unless tag_name&.persisted?
raise ActiveRecord::RecordNotSaved, 'Generated tag name was not persisted'
end
tag_name
end
end
end
+56
ファイルの表示
@@ -0,0 +1,56 @@
module TagNameTranscription
class Unconvertible < StandardError
;
end
def self.ainu name
previous_script = nil
parts = []
NipponeseSegments.split(name).each do |script, text|
output = transcribe_segment(script, text)
if (parts.last && output.first && previous_script && script != previous_script &&
NipponeseSegments.nipponese?(previous_script) && NipponeseSegments.nipponese?(script))
parts << '_'
end
parts.concat(output)
previous_script = script
end
replace_unconvertible(parts)
end
def self.transcribe_segment script, text
return [text] if script == :literal
return [nil] if script == :unconvertible
return transcribe_reading(text) unless script == :kanzi
text.each_char.flat_map do |character|
transcribe_reading(JisReadings.read(character))
rescue Unconvertible
[nil]
end
end
def self.transcribe_reading reading
Phonology.adapt(reading).map do |sound|
sound && Katakana.render([sound])
rescue Unconvertible
nil
end
end
def self.replace_unconvertible parts
grouped = parts.chunk { _1.nil? }.flat_map { |unconvertible, group|
unconvertible ? [nil] : group
}
first = grouped.index { _1&.match?(/[^_]/) }
last = grouped.rindex { _1&.match?(/[^_]/) }
grouped.each_with_index.map { |part, index|
part || (first && first < index && index < last ? '_' : '')
}.join
end
private_class_method :transcribe_segment, :transcribe_reading, :replace_unconvertible
end
+13
ファイルの表示
@@ -0,0 +1,13 @@
module TagNameTranscription
module JisReadings
SOURCE = Rails.root.join('config/tag_name_readings/jis_level_one.tsv')
READINGS = File.foreach(SOURCE, chomp: true).reject { _1.start_with?('#') }.to_h { |line|
character, reading, = line.split("\t")
[character, reading]
}.freeze
def self.read text
text.each_char.map { READINGS.fetch(_1) { raise Unconvertible } }.join
end
end
end
+26
ファイルの表示
@@ -0,0 +1,26 @@
module TagNameTranscription
module Katakana
ROWS = {
'' => 'アイウエオ',
'k' => 'カキクケコ',
's' => 'サシスセソ',
'c' => ['チャ', 'チ', 'チュ', 'チェ', 'チョ'],
't' => ['タ', 'ティ', 'トゥ', 'テ', 'ト'],
'n' => 'ナニヌネノ',
'h' => 'ハヒフヘホ',
'p' => 'パピプペポ',
'm' => 'マミムメモ',
'r' => 'ラリルレロ' }.freeze
SYLLABLES = ROWS.flat_map { |onset, letters|
kana = letters.is_a?(String) ? letters.chars : letters
['a', 'i', 'u', 'e', 'o'].zip(kana).map { |vowel, letter| [onset + vowel, letter] }
}.to_h.merge(
'ya' => 'ヤ', 'yu' => 'ユ', 'yo' => 'ヨ', 'wa' => 'ワ',
'k' => 'ㇰ', 's' => 'ㇱ', 't' => 'ッ', 'n' => 'ン', 'm' => 'ㇺ', 'p' => 'ㇷ゚',
'r:a' => 'ㇻ', 'r:i' => 'ㇼ', 'r:u' => 'ㇽ', 'r:e' => 'ㇾ', 'r:o' => 'ㇿ').freeze
def self.render sounds
sounds.map { SYLLABLES.fetch(_1) { raise Unconvertible } }.join
end
end
end
+30
ファイルの表示
@@ -0,0 +1,30 @@
module TagNameTranscription
module NipponeseSegments
NIPPONESE_SCRIPTS = [:kanzi, :xiragana, :katakana].freeze
def self.nipponese?(script) = NIPPONESE_SCRIPTS.include?(script)
def self.split name
segments = []
name.to_s.scan(/\X/).each do |character|
script = classify(character, segments.last&.first)
if segments.last&.first == script
segments.last[1] << character
else
segments << [script, character.dup]
end
end
segments
end
def self.classify character, previous_script
return previous_script if character == 'ー' && nipponese?(previous_script)
return :kanzi if character.match?(/\A\p{Han}\z/)
return :xiragana if character.match?(/\A\p{Hiragana}\p{M}*\z/)
return :katakana if character.match?(/\A\p{Katakana}\p{M}*\z/)
return :literal if character.match?(/\A(?:\p{Latin}\p{M}*|[!-~])\z/)
:unconvertible
end
end
end
+59
ファイルの表示
@@ -0,0 +1,59 @@
module TagNameTranscription
module Phonology
ROWS = {
'アイウエオ' => ['a', 'i', 'u', 'e', 'o'],
'カキクケコ' => ['ka', 'ki', 'ku', 'ke', 'ko'],
'ガギグゲゴ' => ['ka', 'ki', 'ku', 'ke', 'ko'],
'サシスセソ' => ['sa', 'si', 'su', 'se', 'so'],
'ザジズゼゾ' => ['ca', 'ci', 'cu', 'ce', 'co'],
'タチツテト' => ['ta', 'ci', 'cu', 'te', 'to'],
'ダヂヅデド' => ['ta', 'ci', 'cu', 'te', 'to'],
'ナニヌネノ' => ['na', 'ni', 'nu', 'ne', 'no'],
'ハヒフヘホ' => ['ha', 'hi', 'hu', 'he', 'ho'],
'バビブベボ' => ['pa', 'pi', 'pu', 'pe', 'po'],
'パピプペポ' => ['pa', 'pi', 'pu', 'pe', 'po'],
'マミムメモ' => ['ma', 'mi', 'mu', 'me', 'mo'],
'ヤユヨ' => ['ya', 'yu', 'yo'],
'ラリルレロ' => ['ra', 'ri', 'ru', 're', 'ro'],
'ワ' => ['wa'],
'ン' => ['n'],
'ッ' => ['t'],
'ㇰ' => ['k'],
'ㇱㇲ' => ['s', 's'],
'ㇳ' => ['t'],
'ㇴ' => ['n'],
'ㇺ' => ['m'],
'ㇻㇼㇽㇾㇿ' => ['r:a', 'r:i', 'r:u', 'r:e', 'r:o'] }.freeze
SOUNDS = ROWS.flat_map { |characters, sounds| characters.chars.zip(sounds) }.to_h.freeze
DIGRAPHS = {
'シャ' => 'sa', 'シュ' => 'su', 'シェ' => 'se', 'ショ' => 'so',
'チャ' => 'ca', 'チュ' => 'cu', 'チェ' => 'ce', 'チョ' => 'co',
'ジャ' => 'ca', 'ジュ' => 'cu', 'ジェ' => 'ce', 'ジョ' => 'co',
'ヂャ' => 'ca', 'ヂュ' => 'cu', 'ヂェ' => 'ce', 'ヂョ' => 'co' }.freeze
def self.adapt reading
kana = reading.unicode_normalize(:nfc).tr('ぁ-ゖ', 'ァ-ヶ').delete('ー')
characters = kana.scan(/\X/)
sounds = []
index = 0
while index < characters.length
pair = characters[index, 2].join
if DIGRAPHS.key?(pair)
sounds << DIGRAPHS.fetch(pair)
index += 2
next
end
character = characters[index].tr('ャュョ', 'ヤユヨ')
sounds << (character == 'ㇷ゚' ? 'p' : SOUNDS[character])
index += 1
end
sounds
end
end
end
+22 -6
ファイルの表示
@@ -1,6 +1,9 @@
class TagVersionRecorder < VersionRecorder
def self.record! tag:, event_type:, created_by_user:, **_options
new(tag:, event_type:, created_by_user:).record!
Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
new(tag:, event_type:, created_by_user:).record!
end
end
def initialize tag:, event_type:, created_by_user:
@@ -15,26 +18,39 @@ class TagVersionRecorder < VersionRecorder
def snapshot_attributes
names = snapshot_names
compatibility_names = names.fetch('ja')
compatibility_names = names['ja'] || names['ain'] || names.values.first
unless compatibility_names
@record.errors.add(:tag_names, :invalid)
raise ActiveRecord::RecordInvalid, @record
end
parent_ids =
TagImplication.where(tag_id: @record.id).order(:parent_tag_id)
.lock.pluck(:parent_tag_id)
{ name: compatibility_names.fetch('name'),
category: @record.category,
deprecated_at: @record.deprecated_at,
aliases: compatibility_names.fetch('aliases').map { _1.fetch('name') }.join(' '),
names:,
parent_tag_ids: @record.snapshot_parent_tag_ids.join(' ') }
parent_tag_ids: parent_ids.join(' ') }
end
def snapshot_names
@record.tag_names.order(:language_code, :name).group_by(&:language_code)
@record.tag_names.order(:language_code, :name).lock.group_by(&:language_code)
.transform_values do |tag_names|
primary = tag_names.find(&:primary?)
unless primary
@record.errors.add(:tag_names, :invalid,
language_code: tag_names.first.language_code)
raise ActiveRecord::RecordInvalid, @record
end
aliases = tag_names.reject(&:primary?).sort_by(&:name).map { |tag_name|
{ 'name' => tag_name.name, 'script_code' => tag_name.script_code }
{ 'name' => tag_name.name,
'script_code' => Script.codes.fetch(tag_name.script_code, tag_name.script_code) }
}
{ 'name' => primary.name,
'script_code' => primary.script_code,
'script_code' => Script.codes.fetch(primary.script_code, primary.script_code),
'auto_generated' => primary.auto_generated,
'aliases' => aliases }
end
+13 -4
ファイルの表示
@@ -4,14 +4,23 @@ class TagVersioning
end
def self.ensure_snapshot! tag, created_by_user:, language_code: 'ja'
return if tag.tag_versions.exists?
Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
locked_tag = Tag.unscoped.lock.find(tag.id)
next if locked_tag.tag_versions.lock.exists?
TagVersionRecorder.record!(tag:, event_type: :create, created_by_user:, language_code:)
TagVersionRecorder.record!(
tag: locked_tag, event_type: :create, created_by_user:, language_code:)
end
end
def self.record_tag_snapshot! tag, created_by_user:, language_code: 'ja'
event_type = tag.tag_versions.exists? ? :update : :create
record!(tag, event_type:, created_by_user:, language_code:)
Tag.transaction do
SharedResourceIdAllocator.lock_space!(Tag, 'tags')
locked_tag = Tag.unscoped.lock.find(tag.id)
event_type = locked_tag.tag_versions.lock.exists? ? :update : :create
record!(locked_tag, event_type:, created_by_user:, language_code:)
end
end
def self.record_tag_snapshots! tags, created_by_user:, language_code: 'ja'
+1 -1
ファイルの表示
@@ -35,7 +35,7 @@ class VersionRecorder
private
def latest_version = versions.order(version_no: :desc).first
def latest_version = versions.order(version_no: :desc).lock.first
def versions = @record.public_send(version_association)