diff --git a/backend/app/controllers/materials_controller.rb b/backend/app/controllers/materials_controller.rb index 7c16a1a..93b2fbe 100644 --- a/backend/app/controllers/materials_controller.rb +++ b/backend/app/controllers/materials_controller.rb @@ -255,7 +255,8 @@ class MaterialsController < ApplicationController end def resolve_material_tag! locale, tag_name_raw - Tag.find_or_create_by_tag_name!(locale, tag_name_raw, category: :material) + Tag.find_or_create_by_tag_name!(locale, tag_name_raw, category: :material, + created_by_user: current_user) end def material_index_needs_tag_name? filters diff --git a/backend/app/controllers/nico_tags_controller.rb b/backend/app/controllers/nico_tags_controller.rb index 86a3af9..902358d 100644 --- a/backend/app/controllers/nico_tags_controller.rb +++ b/backend/app/controllers/nico_tags_controller.rb @@ -98,7 +98,8 @@ class NicoTagsController < ApplicationController ApplicationRecord.transaction do linked_tags = Tag.normalise_tags!(locale, linked_tag_names, with_tagme: false, - with_no_deerjikist: false) + with_no_deerjikist: false, + created_by_user: current_user) TagVersioning.record_tag_snapshots!( linked_tags, @@ -140,10 +141,23 @@ class NicoTagsController < ApplicationController end def render_nico_tag_form_record_invalid record - if record.is_a?(TagName) || record.is_a?(Tag) - render_validation_error fields: { tags: record.errors.full_messages.map { |message| - "タグ名 “#{ record.name }”: #{ message }" - } } + names = + case record + when TagName + [record] + when Tag + record.tag_names.to_a + else + [] + end + messages = names.flat_map do |tag_name| + tag_name.errors.full_messages.map do |message| + "タグ名 “#{ tag_name.name }”: #{ message }" + end + end + + if messages.any? + render_validation_error fields: { tags: messages } else render_validation_error record end diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 4593052..9513e97 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -818,7 +818,8 @@ class PostsController < ApplicationController with_tagme: false, with_no_deerjikist: false, deny_deprecated: true, - with_sections: true) => { tags:, sections: } + with_sections: true, + created_by_user: current_user) => { tags:, sections: } tags = Tag.expand_parent_tags(tags).reject(&:deprecated?) video_ms = normalise_video_ms(tags, video_ms_param:, duration_param:) @@ -964,7 +965,8 @@ class PostsController < ApplicationController Tag.normalise_tags!(locale, snapshot[:tag_names], with_tagme: false, deny_deprecated: true, - with_sections: true) => { tags:, sections: } + with_sections: true, + created_by_user: current_user) => { tags:, sections: } TagVersioning.record_tag_snapshots!(tags, created_by_user: current_user, language_code: locale.language_code) diff --git a/backend/app/controllers/sitemap_entries_controller.rb b/backend/app/controllers/sitemap_entries_controller.rb new file mode 100644 index 0000000..e681f81 --- /dev/null +++ b/backend/app/controllers/sitemap_entries_controller.rb @@ -0,0 +1,57 @@ +class SitemapEntriesController < ApplicationController + PAGE_SIZE = 1_000 + RESOURCES = [ + 'posts', 'tags', 'deerjikists', 'materials', + 'descriptions', 'tag_groups', 'theatres'].freeze + + def index + locale = resolve_locale! + resources = RESOURCES.map { |resource| { resource:, count: entries(resource, locale).count } } + render json: { resources:, page_size: PAGE_SIZE } + end + + def show + locale = resolve_locale! + resource = params[:resource] + return head :not_found unless RESOURCES.include?(resource) + + page = Integer(params.fetch(:page, '1'), exception: false) + return head :bad_request unless page && page.positive? + + scope = entries(resource, locale) + return head :not_found if page > [(scope.count.to_f / PAGE_SIZE).ceil, 1].max + + ids = scope.order(:id).offset((page - 1) * PAGE_SIZE).limit(PAGE_SIZE).pluck(:id) + render json: { ids: } + end + + private + + def entries resource, locale + named_tags = TagName.where(language_code: locale.language_code, primary_flg: true) + .select(:tag_id) + case resource + when 'posts' + Post.all + when 'tags' + Tag.where(id: named_tags) + when 'deerjikists' + Tag.where(id: named_tags).where(id: Deerjikist.select(:tag_id)) + when 'materials' + Material.all + when 'descriptions' + TagDescription.where(language_code: locale.language_code, tag_id: named_tags) + .where.not(body: [nil, '']) + .where(TagDescriptionVersion + .where('tag_description_versions.tag_id = tag_descriptions.tag_id') + .where('tag_description_versions.language_code = tag_descriptions.language_code') + .where('tag_description_versions.version_no = tag_descriptions.version_no') + .arel.exists) + when 'tag_groups' + TagGroup.joins(:versions) + .where('tag_group_versions.version_no = tag_groups.version_no') + when 'theatres' + Theatre.kept + end + end +end diff --git a/backend/app/controllers/tags_controller.rb b/backend/app/controllers/tags_controller.rb index 42a478a..7c6feac 100644 --- a/backend/app/controllers/tags_controller.rb +++ b/backend/app/controllers/tags_controller.rb @@ -844,7 +844,8 @@ class TagsController < ApplicationController def update_parent_tags! locale, tag, parent_names parent_tags = Tag.normalise_tags!(locale, parent_names, with_tagme: false, - with_no_deerjikist: false) + with_no_deerjikist: false, + created_by_user: current_user) old_parent_tags = tag.parents.to_a diff --git a/backend/app/models/tag.rb b/backend/app/models/tag.rb index 5336c01..1f66e4a 100644 --- a/backend/app/models/tag.rb +++ b/backend/app/models/tag.rb @@ -142,7 +142,8 @@ class Tag < ApplicationRecord with_tagme: true, with_no_deerjikist: true, deny_deprecated: false, - with_sections: false + with_sections: false, + created_by_user: nil if tag_names.any? { |n| n.downcase.start_with?('nico:') } raise NicoTagNormalisationError end @@ -171,7 +172,8 @@ class Tag < ApplicationRecord name = TagName.canonicalise(locale, name).first - find_or_create_by_tag_name!(locale, name, category: (cat || :general)).tap do |tag| + find_or_create_by_tag_name!(locale, name, category: (cat || :general), + created_by_user:).tap do |tag| if deny_deprecated && tag.deprecated? raise DeprecatedTagNormalisationError, [tag.name(locale.language_code)] end @@ -259,7 +261,7 @@ class Tag < ApplicationRecord [left_end_ms, right_end_ms].max end - def find_or_create_by_tag_name! locale, name, category: + def find_or_create_by_tag_name! locale, name, category:, created_by_user: nil language_code = locale.language_code name = name.to_s.strip @@ -268,11 +270,10 @@ class Tag < ApplicationRecord tn = TagName.lock.find_by(language_code:, name:) next tn.tag if tn - tag = Tag.create!(category:) - TagName.create!(tag:, language_code:, name:, - script_code: locale.script_code, primary_flg: true, - auto_generated: false) - TagNameCompletion.for_tag!(tag) + tag = Tag.new(category:) + tag.tag_names.build(language_code:, name:, script_code: locale.script_code, + primary_flg: true, auto_generated: false) + TagNameCompletion.for_tag!(tag, created_by_user:) tag.association(:tag_names).reset tag diff --git a/backend/app/services/post_creator.rb b/backend/app/services/post_creator.rb index fe3e6a1..2b76334 100644 --- a/backend/app/services/post_creator.rb +++ b/backend/app/services/post_creator.rb @@ -105,7 +105,7 @@ class PostCreator category = spec[:category] || spec['category'] next if name.blank? || category.blank? - tag = Tag.find_or_create_by_tag_name!(locale, name, category:) + tag = Tag.find_or_create_by_tag_name!(locale, name, category:, created_by_user: @actor) tag.update!(category:) if tag.category.to_sym != category.to_sym tags[name] ||= tag end.values diff --git a/backend/app/services/tag_name_completion.rb b/backend/app/services/tag_name_completion.rb index b9fec45..67049ce 100644 --- a/backend/app/services/tag_name_completion.rb +++ b/backend/app/services/tag_name_completion.rb @@ -11,13 +11,14 @@ class TagNameCompletion end Tag.where(id: certified_names.keys).lock.find_each(order: :asc) do |tag| for_tag!(tag, locales: [locale], - certified_names: { locale.language_code => certified_names.fetch(tag.id) }) + certified_names: { locale.language_code => certified_names.fetch(tag.id) }, + created_by_user: nil) end named_tags = TagName.where(language_code: locale.language_code, primary_flg: true) .select(:tag_id) Tag.where.not(id: named_tags).lock.find_each(order: :asc) do |tag| - for_tag!(tag, locales: [locale]) + for_tag!(tag, locales: [locale], created_by_user: nil) end true end @@ -26,12 +27,15 @@ class TagNameCompletion end end - def for_tag! tag, locales: nil, certified_names: { } + def for_tag! tag, locales: nil, certified_names: { }, created_by_user: nil locales ||= Locale.where(deprecated_at: nil) .order(:language_code, :created_at, :script_code).lock Tag.transaction do SharedResourceIdAllocator.lock_space!(Tag, 'tags') + # Creation includes every language in its first snapshot. + creating = tag.new_record? + tag.save! if creating completed = tag.with_lock do primary_names = tag.tag_names.where(primary_flg: true).lock.index_by(&:language_code) certified_names.each do |language, name| @@ -46,9 +50,9 @@ class TagNameCompletion missing_locales = locales.to_a.uniq(&:language_code).reject { |locale| primary_names.key?(locale.language_code) } - next true if missing_locales.empty? + next true if missing_locales.empty? && !creating - TagVersioning.ensure_snapshot!(tag, created_by_user: nil) + TagVersioning.ensure_snapshot!(tag, created_by_user:) unless creating missing_locales.each do |locale| primary_names[locale.language_code] = if certified_names.key?(locale.language_code) @@ -60,7 +64,8 @@ class TagNameCompletion end end tag.association(:tag_names).reset - version = TagVersioning.record!(tag, event_type: :update, created_by_user: nil) + event_type = creating ? :create : :update + version = TagVersioning.record!(tag, event_type:, created_by_user:) unless version&.persisted? raise ActiveRecord::RecordNotSaved, 'Tag version was not persisted' end diff --git a/backend/config/routes.rb b/backend/config/routes.rb index 4e04330..01e0895 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -1,5 +1,6 @@ Rails.application.routes.draw do resources :locales, only: [:index] + resources :sitemap_entries, only: [:index, :show], param: :resource resources :tag_groups, only: [:index, :show, :create, :update] diff --git a/backend/spec/models/locale_spec.rb b/backend/spec/models/locale_spec.rb index 59dcba2..5a8c677 100644 --- a/backend/spec/models/locale_spec.rb +++ b/backend/spec/models/locale_spec.rb @@ -57,6 +57,7 @@ RSpec.describe Locale, type: :model do tag = create(:tag, primary_name: 'already_named') existing = create(:tag_name, name: 'existing_ainu', tag:, language_code: 'ain', script_code: 'Kana') + versions_before = tag.tag_versions.order(:version_no).map(&:attributes) create_ainu_locale! @@ -65,6 +66,7 @@ RSpec.describe Locale, type: :model do expect(existing.reload).to have_attributes( tag_id: tag.id, language_code: 'ain', script_code: 'Kana', primary_flg: true, auto_generated: false) + expect(tag.tag_versions.order(:version_no).map(&:attributes)).to eq(versions_before) end it 'adds the new language primary while preserving existing names and aliases' do @@ -88,6 +90,7 @@ RSpec.describe Locale, type: :model do prepare_ainu_reference! first = create(:tag, primary_name: 'atomic_first') second = create(:tag, primary_name: 'atomic_second') + versions_before = TagVersion.order(:id).map(&:attributes) allow(TagName).to receive(:generate_name).and_call_original allow(TagName).to receive(:generate_name) .with(kind_of(described_class), second, primary_tag_name_for(second, 'ja')) @@ -97,6 +100,69 @@ RSpec.describe Locale, type: :model do expect(described_class.exists?(code: 'ain-Kana')).to be(false) expect(TagName.where(language_code: 'ain', tag_id: [first.id, second.id])).to be_empty + expect(TagVersion.order(:id).map(&:attributes)).to eq(versions_before) + end + + it 'records a separate automatic update for each tag without changing its authored create' do + author = create(:user, :admin) + tags = ['locale_history_first', 'locale_history_second'].map do |name| + Tag.find_or_create_by_tag_name!(Locale.nipponese, name, category: :general, + created_by_user: author) + end + originals = tags.map { |tag| tag.tag_versions.sole.attributes } + + expect { create_ainu_locale! }.to change(TagVersion, :count).by(2) + + tags.zip(originals).each do |tag, original| + versions = tag.tag_versions.order(:version_no).to_a + expect(versions.map(&:event_type)).to eq(['create', 'update']) + expect(versions.first.attributes).to eq(original) + expect(versions.first.created_by_user_id).to eq(author.id) + expect(versions.last).to have_attributes(version_no: 2, created_by_user_id: nil) + expect(versions.last.names.fetch('ain')).to eq( + 'name' => tag.name('ja'), 'script_code' => 'Kana', + 'auto_generated' => true, 'aliases' => []) + expect(versions.first.names.keys).to eq(['ja']) + expect(tag.reload.version_no).to eq(2) + end + + expect { TagNameCompletion.for_locale!(Locale.find('ain-Kana')) } + .not_to change(TagVersion, :count) + end + + it 'records automatic before and after snapshots for a tag without any history' do + tag = create(:tag, primary_name: 'legacy_before_locale') + expect(tag.tag_versions).to be_empty + + create_ainu_locale! + + versions = tag.tag_versions.order(:version_no).to_a + expect(versions.map(&:event_type)).to eq(['create', 'update']) + expect(versions.map(&:created_by_user_id)).to eq([nil, nil]) + expect(versions.first.names.keys).to eq(['ja']) + expect(versions.last.names.keys).to match_array(['ja', 'ain']) + expect(versions.last.names.fetch('ja')).to eq(versions.first.names.fetch('ja')) + end + + it 'rolls back earlier tag names and histories when a later history fails' do + first = create(:tag, primary_name: 'history_atomic_first') + second = create(:tag, primary_name: 'history_atomic_second') + names_before = TagName.order(:id).map(&:attributes) + versions_before = TagVersion.order(:id).map(&:attributes) + allow(TagVersioning).to receive(:record!).and_wrap_original do |original, tag, **options| + if tag.id == second.id && options[:event_type] == :update + raise ActiveRecord::RecordNotSaved, 'history failed' + end + original.call(tag, **options) + end + + expect { create_ainu_locale! } + .to raise_error(ActiveRecord::RecordNotSaved, 'history failed') + + expect(described_class.exists?(code: 'ain-Kana')).to be(false) + expect(TagName.order(:id).map(&:attributes)).to eq(names_before) + expect(TagVersion.order(:id).map(&:attributes)).to eq(versions_before) + expect([first.reload.version_no, second.reload.version_no]).to eq([1, 1]) end it 'rejects a private locale code outside the supported locale set' do diff --git a/backend/spec/requests/locale_write_paths_spec.rb b/backend/spec/requests/locale_write_paths_spec.rb index 51577d4..ab5e75a 100644 --- a/backend/spec/requests/locale_write_paths_spec.rb +++ b/backend/spec/requests/locale_write_paths_spec.rb @@ -14,7 +14,19 @@ RSpec.describe 'Locale propagation on write paths', type: :request do expect(tag_name).to have_attributes( script_code: expected_locale.script_code, primary_flg: true) expect(tag_name.tag_id).to be_present - tag_name.tag + tag = tag_name.tag + version = tag.tag_versions.sole + expect(version).to have_attributes( + event_type: 'create', version_no: 1, created_by_user_id: member.id) + expect(tag.reload.version_no).to eq(1) + expect(version.names.keys).to match_array( + Locale.where(deprecated_at: nil).distinct.pluck(:language_code)) + tag.tag_names.where(primary_flg: true).each do |primary| + expect(version.names.fetch(primary.language_code)).to include( + 'name' => primary.name, 'script_code' => primary.script_code, + 'auto_generated' => primary.auto_generated, 'aliases' => []) + end + tag end it 'confirms a generated Nipponese primary when a human renames it' do diff --git a/backend/spec/requests/nico_tags_spec.rb b/backend/spec/requests/nico_tags_spec.rb index 50fdd0d..a554d26 100644 --- a/backend/spec/requests/nico_tags_spec.rb +++ b/backend/spec/requests/nico_tags_spec.rb @@ -231,11 +231,14 @@ RSpec.describe 'NicoTags', type: :request do replacement: 'valid' ) nico_tag + before_counts = [Tag.count, TagName.count, TagVersion.count] expect { patch "/tags/nico/#{nico_tag.id}", params: { tags: 'created_first invalid' } }.not_to change(TagName, :count) + expect([Tag.count, TagName.count, TagVersion.count]).to eq(before_counts) + expect(response).to have_http_status(:unprocessable_entity) expect(json.fetch('errors').fetch('tags')).to include( a_string_including('タグ名 “invalid”:', '名前に使用できない文字が含まれてゐます.')) diff --git a/backend/spec/requests/sitemap_entries_spec.rb b/backend/spec/requests/sitemap_entries_spec.rb new file mode 100644 index 0000000..26d63cf --- /dev/null +++ b/backend/spec/requests/sitemap_entries_spec.rb @@ -0,0 +1,155 @@ +require 'rails_helper' + +RSpec.describe 'Sitemap entries API', type: :request do + include_context 'Ainu locale' + + let(:user) { create(:user, :member) } + + it 'serves public resource counts with the default locale and page size' do + create(:post) + create(:tag, primary_name: 'sitemap_count_tag') + + get '/sitemap_entries' + + expect(response).to have_http_status(:ok) + expect(json.fetch('page_size')).to eq(1_000) + counts = json.fetch('resources').to_h { |entry| + [entry.fetch('resource'), entry.fetch('count')] + } + expect(counts).to eq( + 'posts' => Post.count, 'tags' => Tag.count, 'deerjikists' => 0, + 'materials' => 0, 'descriptions' => 0, 'tag_groups' => 0, 'theatres' => 0) + end + + it 'paginates public post IDs in ascending order without exposing post contents' do + stub_const('SitemapEntriesController::PAGE_SIZE', 2) + posts = create_list(:post, 3) + + get '/sitemap_entries/posts' + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => posts.first(2).map(&:id)) + + get '/sitemap_entries/posts', params: { page: 2 } + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => [posts.last.id]) + + get '/sitemap_entries/posts', params: { page: 3 } + expect(response).to have_http_status(:not_found) + end + + ['0', '-1', 'invalid', '1.5'].each do |page| + it "rejects invalid sitemap page #{ page.inspect }" do + get '/sitemap_entries/posts', params: { page: } + + expect(response).to have_http_status(:bad_request) + end + end + + it 'rejects unsupported resource names' do + get '/sitemap_entries/users' + + expect(response).to have_http_status(:not_found) + end + + it 'returns an empty first page for a resource without entries' do + get '/sitemap_entries/posts' + + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => []) + end + + it 'rejects unregistered, deprecated and unsupported locales' do + get '/sitemap_entries', params: { locale: 'ain-Kana' } + expect(response).to have_http_status(:not_found) + + locale.update!(deprecated_at: Time.current) + ['/sitemap_entries', '/sitemap_entries/posts'].each do |path| + get path, params: { locale: 'ain-Kana' } + expect(response).to have_http_status(:not_found) + + get path, params: { locale: 'en' } + expect(response).to have_http_status(:not_found) + end + end + + it 'only lists live materials and theatres' do + material = Material.create!(url: 'https://example.com/sitemap-material') + discarded_material = Material.create!(url: 'https://example.com/sitemap-discarded') + discarded_material.discard! + theatre = create(:theatre) + create(:theatre).discard! + + get '/sitemap_entries/materials' + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => [material.id]) + + get '/sitemap_entries/theatres' + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => [theatre.id]) + end + + context 'with an active Ainu locale' do + before { locale } + + it 'uses tag IDs and only includes deerjikists with linked accounts' do + tagged = create(:tag, primary_name: 'sitemap_deer', category: :deerjikist) + unlinked = create(:tag, primary_name: 'sitemap_unlinked', category: :deerjikist) + Deerjikist.create!(tag: tagged, platform: 'youtube', code: 'sitemap-channel') + Deerjikist.create!(tag: tagged, platform: 'nico', code: 'sitemap-nico') + + get '/sitemap_entries/tags', params: { locale: 'ain-Kana' } + expect(response).to have_http_status(:ok) + expect(json.fetch('ids')).to contain_exactly(tagged.id, unlinked.id) + + get '/sitemap_entries/deerjikists', params: { locale: 'ain-Kana' } + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => [tagged.id]) + end + + it 'lists description IDs for the requested language with a nonempty current version' do + tag = create(:tag, primary_name: 'sitemap_description') + japanese = TagDescriptionOperation.create!( + Locale.nipponese, tag:, body: 'Japanese sitemap body', created_by_user: user) + ainu = TagDescriptionOperation.create!( + locale, tag:, body: 'Ainu sitemap body', created_by_user: user) + unversioned_tag = create(:tag, primary_name: 'sitemap_unversioned') + TagDescription.create!(tag: unversioned_tag, language_code: 'ain', + body: 'No revision', version_no: 1) + empty_tag = create(:tag, primary_name: 'sitemap_empty') + # Legacy empty bodies can exist even though the write service rejects them. + empty = TagDescription.create!(tag: empty_tag, language_code: 'ain', + body: '', version_no: 1) + TagDescriptionVersion.create!(tag_id: empty_tag.id, language_code: 'ain', + body: '', version_no: 1, event_type: 'create', + created_by_user_id: user.id, created_at: Time.current) + + get '/sitemap_entries/descriptions', params: { locale: 'ain-Kana' } + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => [ainu.id]) + expect(json.fetch('ids')).not_to include(japanese.id, empty.id) + + get '/sitemap_entries/descriptions', params: { locale: 'ja' } + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => [japanese.id]) + + get '/sitemap_entries', params: { locale: 'ain-Kana' } + expect(response).to have_http_status(:ok) + descriptions = json.fetch('resources').find { |entry| + entry.fetch('resource') == 'descriptions' + } + expect(descriptions.fetch('count')).to eq(1) + end + + it 'excludes groups without a revision matching the current version' do + group = TagGroupOperation.create!( + Locale.nipponese, name: 'Sitemap group', description: 'Group description', + members: [], placements: [], actor: user) + TagGroup.create!(name: 'Unversioned sitemap group', version_no: 1) + + get '/sitemap_entries/tag_groups', params: { locale: 'ain-Kana' } + + expect(response).to have_http_status(:ok) + expect(json).to eq('ids' => [group.id]) + end + end +end diff --git a/backend/spec/services/post_creator_spec.rb b/backend/spec/services/post_creator_spec.rb index 91e6c11..e689f96 100644 --- a/backend/spec/services/post_creator_spec.rb +++ b/backend/spec/services/post_creator_spec.rb @@ -95,7 +95,8 @@ RSpec.describe PostCreator do post = described_class.new(actor:, attributes:).create!(locale) expect(Tag).to have_received(:find_or_create_by_tag_name!) - .with(locale, 'locale_character', category: :character).at_least(:once) + .with(locale, 'locale_character', category: :character, + created_by_user: actor).at_least(:once) tag_name = TagName.find_by!(language_code: 'ain', name: 'locale_character') expect(tag_name).to have_attributes(script_code: 'Kana', primary_flg: true) expect(post.tags).to include(tag_name.tag) diff --git a/backend/spec/services/tag_name_completion_spec.rb b/backend/spec/services/tag_name_completion_spec.rb index ad94730..39f8688 100644 --- a/backend/spec/services/tag_name_completion_spec.rb +++ b/backend/spec/services/tag_name_completion_spec.rb @@ -1,6 +1,7 @@ require 'rails_helper' RSpec.describe TagNameCompletion do + let(:author) { create(:user, :member) } let!(:language) { Language.create!(code: 'ain', name: 'ain') } let!(:script) { Script.create!(code: 'Kana', name: 'Kana') } let(:locale) do @@ -65,4 +66,62 @@ RSpec.describe TagNameCompletion do .to contain_exactly(['ja', 'アイヌ'], ['ain', 'アイヌ']) expect(Tag.find_or_create_by_tag_name!(locale, 'アイヌ', category: :general)).to eq(tag) end + + ['ja', 'ain'].each do |language| + it "records one authored create with all names for a #{ language } origin" do + ainu_locale = locale + origin = language == 'ja' ? Locale.nipponese : ainu_locale + tag = nil + + expect { + tag = Tag.find_or_create_by_tag_name!(origin, 'アイヌ', category: :general, + created_by_user: author) + }.to change(TagVersion, :count).by(1) + + version = tag.tag_versions.sole + expect(version).to have_attributes( + event_type: 'create', version_no: 1, created_by_user_id: author.id) + expect(tag.reload.version_no).to eq(1) + expect(version.names).to eq( + 'ja' => { 'name' => 'アイヌ', 'script_code' => 'Jpan', + 'auto_generated' => language != 'ja', 'aliases' => [] }, + 'ain' => { 'name' => 'アイヌ', 'script_code' => 'Kana', + 'auto_generated' => language != 'ain', 'aliases' => [] }) + + expect { + described_class.for_tag!(tag, created_by_user: author) + found = Tag.find_or_create_by_tag_name!(origin, 'アイヌ', category: :general, + created_by_user: author) + expect(found).to eq(tag) + TagVersioning.record_tag_snapshot!(tag, created_by_user: author) + }.not_to change(TagVersion, :count) + expect(version.reload.created_by_user_id).to eq(author.id) + end + end + + it 'records the author even when no additional language needs completion' do + tag = build(:tag, primary_name: 'single_language_creation') + + described_class.for_tag!(tag, created_by_user: author) + + expect(tag).to be_persisted + expect(tag.tag_versions.sole).to have_attributes( + event_type: 'create', version_no: 1, created_by_user_id: author.id) + expect(tag.tag_versions.sole.names.keys).to eq(['ja']) + end + + it 'rolls back a new tag and all names when its create history fails' do + locale + author + before_counts = [Tag.count, TagName.count, TagVersion.count] + allow(TagVersioning).to receive(:record!).and_raise('history failed') + + expect { + Tag.find_or_create_by_tag_name!(Locale.nipponese, 'atomic_new_history', + category: :general, created_by_user: author) + }.to raise_error('history failed') + + expect([Tag.count, TagName.count, TagVersion.count]).to eq(before_counts) + expect(TagName.where(name: 'atomic_new_history')).to be_empty + end end diff --git a/frontend/server.mjs b/frontend/server.mjs index f4e09b1..9dfcc76 100644 --- a/frontend/server.mjs +++ b/frontend/server.mjs @@ -5,7 +5,7 @@ import { request as requestHTTPS } from 'node:https' import path from 'node:path' import { fileURLToPath } from 'node:url' -import { render } from './dist/server/entry-server.js' +import { render, renderSitemap } from './dist/server/entry-server.js' const root = fileURLToPath (new URL ('./dist/client/', import.meta.url)) const template = await readFile (path.join (root, 'index.html'), 'utf8') @@ -61,6 +61,20 @@ const server = createServer (async (request, response) => { { const url = new URL (request.url, origin) const pathname = decodeURIComponent (url.pathname) + if (pathname === '/sitemap.xml' || pathname.startsWith ('/sitemap-')) + { + const result = await renderSitemap (url.pathname, origin, apiBaseURL) + response.writeHead (result.status, { 'Content-Type': 'application/xml; charset=utf-8' }) + response.end (request.method === 'HEAD' ? undefined : result.body) + return + } + if (pathname === '/robots.txt') + { + response.writeHead (200, { 'Content-Type': 'text/plain; charset=utf-8' }) + response.end (request.method === 'HEAD' + ? undefined : `Sitemap: ${new URL ('/sitemap.xml', origin).href}\n`) + return + } const file = path.resolve (root, '.' + pathname) const allowed = file.startsWith (root) && !pathname.split ('/').some (segment => segment.startsWith ('.')) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 07a1872..d083062 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -124,32 +124,34 @@ export const pageRoutes = ( { path: '', element: , handle: { redirectTo: '/posts' } }, { path: 'posts', element: , - handle: { preload: PostListPage.preload, switchLocale: switchPostFilterLocale } }, + handle: { preload: PostListPage.preload, switchLocale: switchPostFilterLocale, + sitemap: ['static'] } }, { path: 'posts/new', element: , handle: { preload: PostNewPage.preload, robots: 'noindex' } }, { path: 'posts/search', element: , handle: { preload: PostSearchPage.preload, switchLocale: switchPostFilterLocale } }, { path: 'posts/:id', element: , - handle: { preload: PostDetailPage.preload } }, + handle: { preload: PostDetailPage.preload, sitemap: ['posts'] } }, { path: 'posts/changes', element: , handle: { preload: PostHistoryPage.preload } }, - { path: 'tags', element: , handle: { preload: TagListPage.preload } }, + { path: 'tags', element: , + handle: { preload: TagListPage.preload, sitemap: ['static'] } }, { path: 'tags/:id', element: , - handle: { preload: TagDetailPage.preload } }, + handle: { preload: TagDetailPage.preload, sitemap: ['tags'] } }, { path: 'tags/:id/deerjikists', element: , - handle: { preload: DeerjikistDetailPage.preload } }, + handle: { preload: DeerjikistDetailPage.preload, sitemap: ['deerjikists'] } }, { path: 'tags/nico', element: , - handle: { preload: NicoTagListPage.preload } }, + handle: { preload: NicoTagListPage.preload, sitemap: ['static'] } }, { path: 'nico/tags', element: , handle: { preload: NicoTagListPage.preload } }, { path: 'tags/changes', element: , handle: { preload: TagHistoryPage.preload } }, { path: 'theatres/:id', element: , - handle: { preload: TheatreDetailPage.preload } }, + handle: { preload: TheatreDetailPage.preload, sitemap: ['theatres'] } }, { path: 'materials', element: , handle: { preload: MaterialBasePage.preload }, children: [ { index: true, element: , - handle: { preload: MaterialListPage.preload } }, + handle: { preload: MaterialListPage.preload, sitemap: ['static'] } }, { path: 'changes', element: , handle: { preload: MaterialHistoryPage.preload } }, { path: 'new', element: , @@ -158,11 +160,12 @@ export const pageRoutes = ( handle: { preload: MaterialSyncSuppressionsPage.preload, robots: 'noindex', authentication: 'user' } }, { path: ':id', element: , - handle: { preload: MaterialDetailPage.preload } }] }, + handle: { preload: MaterialDetailPage.preload, sitemap: ['materials'] } }] }, { path: 'wiki', element: , - handle: { preload: WikiSearchPage.preload } }, + handle: { preload: WikiSearchPage.preload, sitemap: ['static'] } }, { path: 'wiki/:title', element: , - handle: { preload: WikiDetailPage.preload, switchLocale: switchWikiLocale } }, + handle: { preload: WikiDetailPage.preload, switchLocale: switchWikiLocale, + sitemap: ['descriptions', 'tag_groups'] } }, { path: 'wiki/new', element: , handle: { preload: WikiNewPage.preload, robots: 'noindex' } }, { path: 'wiki/:id/edit', element: , @@ -174,22 +177,25 @@ export const pageRoutes = ( { path: 'wiki/changes', element: , handle: { preload: WikiHistoryPage.preload, switchLocale: request => switchWikiLocale (request, 'history') } }, - { path: 'help', element: , handle: { preload: HelpListPage.preload } }, + { path: 'help', element: , + handle: { preload: HelpListPage.preload, sitemap: ['static'] } }, { path: 'help/changes', element: , handle: { preload: HelpChangesPage.preload } }, { path: 'help/:identifier/diff', element: , handle: { preload: HelpDiffPage.preload } }, { id: 'help.detail', path: 'help/:identifier', element: , - handle: { prepare: prepareHelpPage } }, + handle: { prepare: prepareHelpPage, sitemap: ['help'] } }, { path: 'users/settings', element: , handle: { preload: SettingPage.preload, robots: 'noindex', authentication: 'user' } }, { path: 'settings', element: , handle: { redirectTo: '/users/settings' } }, - { path: 'tos', element: , handle: { preload: TOSPage.preload } }, + { path: 'tos', element: , + handle: { preload: TOSPage.preload, sitemap: ['static'] } }, { id: 'gekanator', path: 'gekanator', element: , - handle: { locale: 'ja', preload: locale => locale === 'ja' + handle: { locale: 'ja', sitemap: ['static'], preload: locale => locale === 'ja' ? GekanatorPage.preload () : GekanatorUnavailablePage.preload () } }, - { path: 'more', element: , handle: { preload: MorePage.preload } }, + { path: 'more', element: , + handle: { preload: MorePage.preload, sitemap: ['static'] } }, { path: '*', element: , handle: { notFound: true } }] diff --git a/frontend/src/components/TopNav.tsx b/frontend/src/components/TopNav.tsx index 5f31ef1..4904b96 100644 --- a/frontend/src/components/TopNav.tsx +++ b/frontend/src/components/TopNav.tsx @@ -483,9 +483,7 @@ const TopNav: FC = ({ user, locales, resolveLocaleSwitch }) => { key={item.to} initial={false} className="shrink-0 whitespace-nowrap" - {...(animationsOff - ? { } - : { layoutId: `menu-${ menu.indexOf (item) }` })} + layoutId={`menu-${ menu.indexOf (item) }`} onLayoutAnimationComplete={() => measure (highlightTargetRef.current)} animate={layoutReady ? { opacity: moreVsbl ? 0 : 1 } : undefined} transition={{ opacity: opacityTransition, @@ -548,8 +546,7 @@ const TopNav: FC = ({ user, locales, resolveLocaleSwitch }) => { = ({ user, locales, resolveLocaleSwitch }) => {
measure (highlightTargetRef.current)} className="z-10 min-w-0 px-3 py-2 font-bold">

@@ -596,13 +592,12 @@ const TopNav: FC = ({ user, locales, resolveLocaleSwitch }) => { key={`c-${ i }-${ j }`} className="min-w-0 max-w-full" transition={topNavTransition} - {...(animationsOff - ? { } - : ((visibleMenu[activeIdx] === item) + {...(visibleMenu[activeIdx] === item ? { layoutId: `submenu-${ menu.indexOf (item) }-${ j }` } - : { initial: { y: -40, opacity: 0 }, - animate: { y: 0, opacity: 1 }, - exit: { y: -40, opacity: 0 } }))}> + : (animationsOff ? { } + : { initial: { y: -40, opacity: 0 }, + animate: { y: 0, opacity: 1 }, + exit: { y: -40, opacity: 0 } }))}> {subItem.component} ) : ( @@ -610,13 +605,12 @@ const TopNav: FC = ({ user, locales, resolveLocaleSwitch }) => { key={`l-${ i }-${ j }`} className="min-w-0 max-w-full" transition={topNavTransition} - {...(animationsOff - ? { } - : ((visibleMenu[activeIdx] === item) + {...(visibleMenu[activeIdx] === item ? { layoutId: `submenu-${ menu.indexOf (item) }-${ j }` } - : { initial: { y: -40, opacity: 0 }, - animate: { y: 0, opacity: 1 }, - exit: { y: -40, opacity: 0 } }))}> + : (animationsOff ? { } + : { initial: { y: -40, opacity: 0 }, + animate: { y: 0, opacity: 1 }, + exit: { y: -40, opacity: 0 } }))}> Promise prepare?: (request: PagePreparation) => Promise @@ -22,6 +26,7 @@ export type PageRouteHandle = { locale?: LocaleCode robots?: 'noindex' authentication?: 'user' + sitemap?: SitemapSource[] redirectTo?: string notFound?: boolean } diff --git a/frontend/src/lib/sitemap.server.ts b/frontend/src/lib/sitemap.server.ts new file mode 100644 index 0000000..e3b7a77 --- /dev/null +++ b/frontend/src/lib/sitemap.server.ts @@ -0,0 +1,152 @@ +import { generatePath } from 'react-router-dom' + +import { pageRoutes } from '@/App' +import { apiGet, isApiError } from '@/lib/api' +import { parseHelpEdition } from '@/lib/help' +import { fetchLocales, isLocaleCode, localeLanguage } from '@/lib/locales' +import { withRenderContext } from '@/lib/renderContext.server' + +import type { LocaleCode } from '@/lib/locales' +import type { PageRoute, SitemapSource } from '@/lib/routeLocales' + +type SitemapRoute = { path: string; source: SitemapSource } +type SitemapResponse = { status: number; body: string } +type ResourceCounts = { + resources: { resource: SitemapSource; count: number }[] + pageSize: number } + +const PAGE_SIZE = 1_000 + + +const sitemapRoutes = (locale: LocaleCode): SitemapRoute[] => { + const collect = (routes: PageRoute[], parent = ''): SitemapRoute[] => + routes.flatMap (route => { + const handle = route.handle + if (handle?.robots === 'noindex' || handle?.authentication != null + || handle?.redirectTo != null || handle?.notFound + || (handle?.locale != null && handle.locale !== locale)) + return [] + const path = [parent, route.path].filter (Boolean).join ('/') + return [ + ...(handle?.sitemap ?? []).map (source => ({ path, source })), + ...collect (route.children ?? [], path)] + }) + return collect (pageRoutes (null, () => { })) +} + + +const helpIdentifiers = async (locale: LocaleCode): Promise => { + const { default: articles } = await import ('virtual:help/articles') + const language = localeLanguage (locale) + return Object.values (articles).filter (article => { + const edition = article.languages[language] + return edition != null && edition.publishedEditions.includes (edition.currentEdition) + && parseHelpEdition (edition.currentEdition) != null + }).map (article => article.identifier).sort () +} + + +const escapeXML = (value: string): string => + value.replace (/&/g, '&').replace (//g, '>') + .replace (/"/g, '"').replace (/'/g, ''') + + +const sitemapXML = (urls: string[], index = false): string => { + if (urls.length > 50_000) + throw new Error ('Sitemap URL limit exceeded') + const root = index ? 'sitemapindex' : 'urlset' + const item = index ? 'sitemap' : 'url' + return '\n' + + `<${root} xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n` + + urls.map (url => ` <${item}>${escapeXML (url)}`).join ('\n') + + `\n\n` +} + + +export const renderSitemap = async ( + address: string, + origin: string, + apiBaseURL: string, +): Promise => + withRenderContext ({ pathname: address, origin, apiBaseURL }, async () => { + const url = new URL (address, origin) + const index = url.pathname === '/sitemap.xml' + const matched = url.pathname.match (/^\/sitemap-(.+)-([a-z_]+)-([1-9]\d*)\.xml$/) + if (!index && matched == null) + return { status: 404, body: '' } + + // Sitemap requests are always public; incoming user cookies are not forwarded. + const available = await fetchLocales () + const locales = available.locales.map (entry => entry.code).filter (isLocaleCode) + if (index) + { + const urls: string[] = [] + for (const locale of locales) + { + const sources = new Set (sitemapRoutes (locale).map (route => route.source)) + const counts = await apiGet ('/sitemap_entries', { locale }) + for (const source of sources) + { + const count = + source === 'static' + ? 1 + : (source === 'help' + ? (await helpIdentifiers (locale)).length + : counts.resources.find (entry => entry.resource === source)?.count ?? 0) + const pageSize = source === 'help' ? PAGE_SIZE : counts.pageSize + if (!Number.isSafeInteger (pageSize) || pageSize <= 0 + || !Number.isSafeInteger (count) || count < 0) + throw new Error ('Invalid sitemap pagination') + const pages = Math.ceil (count / pageSize) + if (urls.length + pages > 50_000) + throw new Error ('Sitemap index limit exceeded') + for (let page = 1; page <= pages; ++page) + urls.push (new URL (`/sitemap-${locale}-${source}-${page}.xml`, origin).href) + } + } + return { status: 200, body: sitemapXML (urls, true) } + } + + const [, locale, source, pageValue] = matched! + const page = Number (pageValue) + if (!isLocaleCode (locale) || !locales.includes (locale) + || !Number.isSafeInteger (page)) + return { status: 404, body: '' } + const routes = sitemapRoutes (locale).filter (route => route.source === source) + if (routes.length === 0) + return { status: 404, body: '' } + + const parameters: Record[] = [] + if (source === 'static') + { + if (page !== 1) + return { status: 404, body: '' } + parameters.push ({ }) + } + else if (source === 'help') + { + const identifiers = await helpIdentifiers (locale) + parameters.push (...identifiers.slice ((page - 1) * PAGE_SIZE, page * PAGE_SIZE) + .map (identifier => ({ identifier: encodeURIComponent (identifier) }))) + } + else + { + try + { + const { ids } = await apiGet<{ ids: number[] }> ( + `/sitemap_entries/${source}`, { locale, params: { page } }) + parameters.push (...ids.map (id => ({ id: String (id), title: String (id) }))) + } + catch (error) + { + if (isApiError (error) && error.response?.status === 404) + return { status: 404, body: '' } + throw error + } + } + if (parameters.length === 0) + return { status: 404, body: '' } + const urls = routes.flatMap (route => parameters.map (params => + new URL (generatePath (`/${locale}/${route.path}`, params), origin).href)) + return { status: 200, body: sitemapXML ([...new Set (urls)]) } + })