#399

backend/spec/requests/post_imports_spec.rb

@@ -0,0 +1,112 @@
+require 'rails_helper'
+
+RSpec.describe 'Post imports API', type: :request do
+  let(:member) { create(:user, :member) }
+
+  before do
+    allow(Preview::UrlSafety).to receive(:validate) do |url|
+      [URI.parse(url), ['8.8.8.8']]
+    end
+    allow(PostMetadataFetcher).to receive(:fetch).and_return(
+      title: 'fetched title',
+      thumbnail_base: nil,
+      tags: ''
+    )
+  end
+
+  describe 'POST /posts/import/preview' do
+    it 'requires a member' do
+      sign_out
+      post '/posts/import/preview', params: { source: 'https://example.com/post' }
+      expect(response).to have_http_status(:unauthorized)
+
+      sign_in_as(create(:user, :guest))
+      post '/posts/import/preview', params: { source: 'https://example.com/post' }
+      expect(response).to have_http_status(:forbidden)
+    end
+
+    it 'parses a URL list and returns preview rows' do
+      sign_in_as(member)
+
+      post '/posts/import/preview', params: {
+        source: " https://example.com/one \r\n\r\nhttps://example.com/two"
+      }
+
+      expect(response).to have_http_status(:ok)
+      expect(json.fetch('rows').map { _1.fetch('source_row') }).to eq([1, 3])
+      expect(json.fetch('rows').map { _1.fetch('url') }).to eq([
+        'https://example.com/one',
+        'https://example.com/two'
+      ])
+    end
+
+    it 'returns a safe 400 response for an invalid source' do
+      sign_in_as(member)
+
+      post '/posts/import/preview', params: { source: '' }
+
+      expect(response).to have_http_status(:bad_request)
+      expect(json.fetch('message')).to eq('URL を入力してください．')
+    end
+  end
+
+  describe 'POST /posts/import/validate' do
+    it 'accepts camel-case row properties and returns their warnings' do
+      sign_in_as(member)
+
+      post '/posts/import/validate', params: {
+        rows: [{
+          sourceRow: '1',
+          url: 'https://example.com/post',
+          metadataUrl: 'https://example.com/post',
+          attributes: { title: 'manual title' },
+          provenance: { url: 'manual', title: 'manual' },
+          tagSources: { automatic: '', manual: '' },
+          fieldWarnings: { title: ['old warning'] },
+          baseWarnings: ['base warning']
+        }],
+        changed_row: -1
+      }
+
+      expect(response).to have_http_status(:ok)
+      result = json.fetch('rows').first
+      expect(result.fetch('source_row')).to eq(1)
+      expect(result.fetch('tag_sources')).to eq('automatic' => '', 'manual' => '')
+      expect(result.fetch('field_warnings')).to eq('title' => ['old warning'])
+      expect(result.fetch('base_warnings')).to eq(['base warning'])
+    end
+
+    it 'rejects a non-array rows value with 400' do
+      sign_in_as(member)
+
+      post '/posts/import/validate', params: { rows: { sourceRow: 1 }, changed_row: -1 }
+
+      expect(response).to have_http_status(:bad_request)
+      expect(json.fetch('message')).to eq('取込行の形式が不正です．')
+    end
+  end
+
+  describe 'POST /posts/import' do
+    it 'returns a formal skipped result for an existing post' do
+      existing = create(:post, url: 'https://example.com/existing')
+      sign_in_as(member)
+
+      post '/posts/import', params: {
+        rows: [{
+          sourceRow: 1,
+          url: existing.url,
+          attributes: { title: 'ignored' },
+          provenance: { url: 'manual', title: 'manual' },
+          tagSources: { automatic: '', manual: '' }
+        }]
+      }
+
+      expect(response).to have_http_status(:ok)
+      expect(json).to include('created' => 0, 'skipped' => 1, 'failed' => 0)
+      expect(json.fetch('rows').first).to include(
+        'status' => 'skipped',
+        'existing_post_id' => existing.id
+      )
+    end
+  end
+end

backend/spec/services/post_import_previewer_spec.rb

@@ -0,0 +1,187 @@
+require 'rails_helper'
+
+RSpec.describe PostImportPreviewer do
+  before do
+    allow(Preview::UrlSafety).to receive(:validate) do |url|
+      [URI.parse(url), ['8.8.8.8']]
+    end
+  end
+
+  def row(source_row:, url:, attributes: { }, provenance: { }, **values)
+    {
+      source_row:,
+      url:,
+      attributes:,
+      provenance:
+    }.merge(values)
+  end
+
+  describe '#preview_rows' do
+    it 'marks an existing post for skipping without fetching metadata' do
+      existing = create(:post, url: 'https://example.com/existing')
+      expect(PostMetadataFetcher).not_to receive(:fetch)
+
+      result = described_class.new.preview_rows(rows: [
+        row(source_row: 1, url: 'https://EXAMPLE.com/existing/')
+      ]).first
+
+      expect(result).to include(
+        url: 'https://example.com/existing',
+        skip_reason: 'existing',
+        existing_post_id: existing.id,
+        validation_errors: {}
+      )
+      expect(result.fetch(:field_warnings)).to eq({})
+    end
+
+    it 'reports list duplicates as URL errors instead of existing skips' do
+      create(:post, url: 'https://example.com/duplicate')
+      expect(PostMetadataFetcher).not_to receive(:fetch)
+
+      results = described_class.new.preview_rows(rows: [
+        row(source_row: 1, url: 'https://example.com/duplicate'),
+        row(source_row: 2, url: 'https://EXAMPLE.com/duplicate/')
+      ])
+
+      expect(results.map { _1[:validation_errors] }).to all(
+        include(url: ['URL が重複しています．'])
+      )
+      expect(results).to all(include(skip_reason: nil, existing_post_id: nil))
+    end
+
+    it 'does not fetch metadata for an unsafe URL' do
+      allow(Preview::UrlSafety).to receive(:validate)
+          .and_raise(Preview::UrlSafety::UnsafeUrl, '安全でない接続先は使用できません．')
+      expect(PostMetadataFetcher).not_to receive(:fetch)
+
+      result = described_class.new.preview_rows(rows: [
+        row(source_row: 1, url: 'https://unsafe.example/post')
+      ]).first
+
+      expect(result.fetch(:validation_errors)).to include(
+        url: ['安全でない接続先は使用できません．']
+      )
+    end
+
+    it 'applies metadata to automatic fields and recognises metadata tags' do
+      Tag.create!(name: 'known-tag', category: :general)
+      allow(PostMetadataFetcher).to receive(:fetch).and_return(
+        title: 'metadata title',
+        thumbnail_base: 'https://example.com/thumb.jpg',
+        duration: 2_000,
+        tags: 'known-tag'
+      )
+
+      result = described_class.new.preview_rows(rows: [
+        row(source_row: 1, url: 'https://example.com/new')
+      ]).first
+
+      expect(result.fetch(:attributes)).to include(
+        'title' => 'metadata title',
+        'thumbnail_base' => 'https://example.com/thumb.jpg',
+        'duration' => 2_000,
+        'tags' => 'known-tag'
+      )
+      expect(result.fetch(:field_warnings)).not_to have_key('tags')
+      expect(PostMetadataFetcher).to have_received(:fetch).once
+    end
+
+    it 'preserves manual empty values when metadata is available' do
+      allow(PostMetadataFetcher).to receive(:fetch).and_return(
+        title: 'metadata title',
+        tags: 'metadata-tag'
+      )
+
+      result = described_class.new.preview_rows(rows: [
+        row(
+          source_row: 1,
+          url: 'https://example.com/manual',
+          attributes: { title: '', tags: '' },
+          provenance: { title: 'manual', tags: 'manual' },
+          tag_sources: { automatic: 'old-tag', manual: '' }
+        )
+      ]).first
+
+      expect(result.fetch(:attributes)).to include('title' => '', 'tags' => '')
+      expect(result.fetch(:tag_sources)).to include('manual' => '')
+    end
+
+    it 'clears warnings derived from an old metadata URL' do
+      allow(PostMetadataFetcher).to receive(:fetch).and_return(
+        title: 'new title',
+        thumbnail_base: 'https://example.com/new-thumb.jpg',
+        tags: ''
+      )
+
+      result = described_class.new.preview_rows(rows: [
+        row(
+          source_row: 1,
+          url: 'https://example.com/new-url',
+          attributes: { title: 'old title', tags: 'old-tag' },
+          provenance: { title: 'automatic', tags: 'automatic' },
+          tag_sources: { automatic: 'old-tag', manual: '' },
+          metadata_url: 'https://example.com/old-url',
+          field_warnings: { tags: ['old tag warning'], url: ['old fetch warning'] },
+          base_warnings: ['old base warning']
+        )
+      ]).first
+
+      expect(result.fetch(:attributes)).to include('title' => 'new title', 'tags' => '')
+      expect(result.fetch(:base_warnings)).to eq([])
+      expect(result.fetch(:field_warnings).values.flatten)
+          .not_to include('old tag warning', 'old fetch warning')
+    end
+
+    it 'isolates an unexpected metadata failure to the affected URL' do
+      allow(PostMetadataFetcher).to receive(:fetch) do |url|
+        raise 'fetch failure' if url.include?('failure')
+
+        { title: 'successful title', tags: '' }
+      end
+      allow(Rails.logger).to receive(:error)
+
+      results = described_class.new.preview_rows(rows: [
+        row(source_row: 1, url: 'https://example.com/failure'),
+        row(source_row: 2, url: 'https://example.com/success')
+      ])
+
+      expect(results[0].fetch(:field_warnings)).to include(
+        'url' => [described_class::METADATA_FETCH_WARNING]
+      )
+      expect(results[1].fetch(:attributes)).to include('title' => 'successful title')
+    end
+
+    it 'turns an unsafe URL detected during metadata fetching into a URL error' do
+      allow(PostMetadataFetcher).to receive(:fetch)
+          .and_raise(Preview::UrlSafety::UnsafeUrl, '安全でない接続先です．')
+      allow(Rails.logger).to receive(:info)
+
+      result = described_class.new.preview_rows(rows: [
+        row(source_row: 1, url: 'https://example.com/redirects-to-private')
+      ]).first
+
+      expect(result.fetch(:validation_errors)).to include(
+        url: ['安全でない接続先です．']
+      )
+    end
+
+    it 'validates parent IDs from the preloaded set' do
+      parent = create(:post)
+
+      results = described_class.new.preview_rows(
+        rows: [
+          row(source_row: 1, url: 'https://example.com/valid-parent',
+              attributes: { parent_post_ids: parent.id.to_s }),
+          row(source_row: 2, url: 'https://example.com/missing-parent',
+              attributes: { parent_post_ids: '999999' })
+        ],
+        fetch_metadata: false
+      )
+
+      expect(results[0].fetch(:validation_errors)).not_to have_key(:parent_post_ids)
+      expect(results[1].fetch(:validation_errors)).to include(
+        parent_post_ids: ['存在しない親投稿 Id. があります．']
+      )
+    end
+  end
+end

backend/spec/services/post_import_row_normaliser_spec.rb

@@ -0,0 +1,77 @@
+require 'rails_helper'
+
+RSpec.describe PostImportRowNormaliser do
+  def valid_row(overrides = { })
+    {
+      sourceRow: '1',
+      url: 'https://example.com/post',
+      metadataUrl: 'https://example.com/post',
+      attributes: { title: 'title', duration: 1_000 },
+      provenance: { url: 'manual', title: 'automatic' },
+      tagSources: { automatic: 'tag', manual: '' }
+    }.deep_merge(overrides)
+  end
+
+  describe '.normalise!' do
+    it 'normalises aliases and source rows into a permitted plain hash' do
+      row = ActionController::Parameters.new(valid_row)
+
+      expect(described_class.normalise!([row])).to eq([
+        {
+          'source_row' => 1,
+          'url' => 'https://example.com/post',
+          'metadata_url' => 'https://example.com/post',
+          'attributes' => { 'title' => 'title', 'duration' => 1_000 },
+          'provenance' => { 'url' => 'manual', 'title' => 'automatic' },
+          'tag_sources' => { 'automatic' => 'tag', 'manual' => '' }
+        }
+      ])
+    end
+
+    it 'normalises source rows before checking duplicates' do
+      rows = [valid_row, valid_row(sourceRow: 1, url: 'https://example.com/other')]
+
+      expect { described_class.normalise!(rows) }
+          .to raise_error(ArgumentError, '元行番号が重複しています．')
+    end
+
+    it 'rejects non-array batches and non-hash rows' do
+      expect { described_class.normalise!({}) }
+          .to raise_error(ArgumentError, '取込行の形式が不正です．')
+      expect { described_class.normalise!(['row']) }
+          .to raise_error(ArgumentError, '取込行の形式が不正です．')
+    end
+
+    it 'rejects unknown attributes and invalid field types' do
+      expect { described_class.normalise!([valid_row(attributes: { unknown: 'x' })]) }
+          .to raise_error(ArgumentError, '取込項目が不正です．')
+      expect { described_class.normalise!([valid_row(attributes: { title: [] })]) }
+          .to raise_error(ArgumentError, '取込項目の型が不正です．')
+      expect { described_class.normalise!([valid_row(attributes: { duration: false })]) }
+          .to raise_error(ArgumentError, '取込項目の型が不正です．')
+    end
+
+    it 'rejects unknown provenance and tag-source values' do
+      expect { described_class.normalise!([valid_row(provenance: { title: 'mapped' })]) }
+          .to raise_error(ArgumentError, '値の由来が不正です．')
+      expect { described_class.normalise!([valid_row(tagSources: { mapped: 'tag' })]) }
+          .to raise_error(ArgumentError, 'タグ由来の形式が不正です．')
+    end
+
+    it 'accepts warning fields only at the validation boundary' do
+      row = valid_row.merge(
+        fieldWarnings: { title: ['取得できませんでした．'] },
+        baseWarnings: ['確認してください．']
+      )
+
+      without_warnings = described_class.normalise!([row]).first
+      with_warnings = described_class.normalise!([row], allow_warning_fields: true).first
+
+      expect(without_warnings).not_to include('field_warnings', 'base_warnings')
+      expect(with_warnings).to include(
+        'field_warnings' => { 'title' => ['取得できませんでした．'] },
+        'base_warnings' => ['確認してください．']
+      )
+    end
+  end
+end

backend/spec/services/post_import_runner_spec.rb

@@ -0,0 +1,133 @@
+require 'rails_helper'
+
+RSpec.describe PostImportRunner do
+  let(:actor) { create(:user, :member) }
+
+  def row(source_row: 1, url: 'https://example.com/post')
+    {
+      sourceRow: source_row,
+      url:,
+      attributes: { title: 'title', tags: '' },
+      provenance: { url: 'manual', title: 'manual', tags: 'manual' },
+      tagSources: { automatic: '', manual: '' }
+    }
+  end
+
+  def preview(source_row: 1, errors: { }, skip_reason: nil, existing_post_id: nil)
+    {
+      source_row:,
+      attributes: { 'title' => 'title', 'tags' => '' },
+      validation_errors: errors,
+      skip_reason:,
+      existing_post_id:
+    }
+  end
+
+  it 'previews the whole batch once before processing individual rows' do
+    rows = [row, row(source_row: 2, url: 'https://example.com/two')]
+    previewer = instance_double(PostImportPreviewer)
+    allow(PostImportPreviewer).to receive(:new).and_return(previewer)
+    expect(previewer).to receive(:preview_rows)
+        .with(
+          rows: satisfy { _1.map { |row_value| row_value['source_row'] } == [1, 2] },
+          fetch_metadata: false
+        )
+        .and_return([preview, preview(source_row: 2)])
+    allow(PostCreator).to receive(:new).and_return(
+      instance_double(PostCreator, create!: create(:post))
+    )
+
+    result = described_class.new(actor:, rows:).run
+
+    expect(result).to include(created: 2, skipped: 0, failed: 0)
+  end
+
+  it 'treats validation errors as failures before an existing skip' do
+    existing = create(:post)
+    previewer = instance_double(PostImportPreviewer)
+    allow(PostImportPreviewer).to receive(:new).and_return(previewer)
+    allow(previewer).to receive(:preview_rows).and_return([
+      preview(errors: { url: ['URL が重複しています．'] },
+              skip_reason: 'existing', existing_post_id: existing.id)
+    ])
+    expect(PostCreator).not_to receive(:new)
+
+    result = described_class.new(actor:, rows: [row]).run.fetch(:rows).first
+
+    expect(result).to include(
+      status: 'failed',
+      errors: { url: ['URL が重複しています．'] },
+      recoverable: true
+    )
+  end
+
+  it 'returns the existing post ID for skipped rows' do
+    existing = create(:post)
+    previewer = instance_double(PostImportPreviewer)
+    allow(PostImportPreviewer).to receive(:new).and_return(previewer)
+    allow(previewer).to receive(:preview_rows).and_return([
+      preview(skip_reason: 'existing', existing_post_id: existing.id)
+    ])
+
+    result = described_class.new(actor:, rows: [row]).run.fetch(:rows).first
+
+    expect(result).to eq(
+      source_row: 1,
+      status: 'skipped',
+      existing_post_id: existing.id
+    )
+  end
+
+  it 'converts a URL uniqueness validation race into a skip' do
+    existing = create(:post, url: 'https://example.com/race')
+    invalid = Post.new(url: existing.url)
+    invalid.errors.add(:url, :taken)
+    previewer = instance_double(PostImportPreviewer)
+    allow(PostImportPreviewer).to receive(:new).and_return(previewer)
+    allow(previewer).to receive(:preview_rows).and_return([preview])
+    creator = instance_double(PostCreator)
+    allow(PostCreator).to receive(:new).and_return(creator)
+    allow(creator).to receive(:create!).and_raise(ActiveRecord::RecordInvalid.new(invalid))
+
+    result = described_class.new(
+      actor:,
+      rows: [row(url: 'https://EXAMPLE.com/race/')]
+    ).run.fetch(:rows).first
+
+    expect(result).to include(status: 'skipped', existing_post_id: existing.id)
+  end
+
+  it 're-raises RecordNotUnique errors unrelated to the posts URL index' do
+    previewer = instance_double(PostImportPreviewer)
+    allow(PostImportPreviewer).to receive(:new).and_return(previewer)
+    allow(previewer).to receive(:preview_rows).and_return([preview])
+    creator = instance_double(PostCreator)
+    allow(PostCreator).to receive(:new).and_return(creator)
+    allow(creator).to receive(:create!)
+        .and_raise(ActiveRecord::RecordNotUnique, 'other_unique_index')
+
+    expect {
+      described_class.new(actor:, rows: [row]).run
+    }.to raise_error(ActiveRecord::RecordNotUnique)
+  end
+
+  it 'converts a posts URL index race into a skip' do
+    existing = create(:post, url: 'https://example.com/index-race')
+    previewer = instance_double(PostImportPreviewer)
+    allow(PostImportPreviewer).to receive(:new).and_return(previewer)
+    allow(previewer).to receive(:preview_rows).and_return([preview])
+    creator = instance_double(PostCreator)
+    allow(PostCreator).to receive(:new).and_return(creator)
+    allow(creator).to receive(:create!).and_raise(
+      ActiveRecord::RecordNotUnique,
+      'duplicate key index_posts_on_url'
+    )
+
+    result = described_class.new(
+      actor:,
+      rows: [row(url: 'https://EXAMPLE.com/index-race/')]
+    ).run.fetch(:rows).first
+
+    expect(result).to include(status: 'skipped', existing_post_id: existing.id)
+  end
+end

backend/spec/services/post_import_url_list_parser_spec.rb

@@ -0,0 +1,40 @@
+require 'rails_helper'
+
+RSpec.describe PostImportUrlListParser do
+  describe '.parse' do
+    it 'trims URLs, ignores blank lines, and preserves source line numbers' do
+      source = "  https://example.com/one  \r\n\r\nhttps://example.com/two\n"
+
+      expect(described_class.parse(source)).to eq([
+        { source_row: 1, url: 'https://example.com/one' },
+        { source_row: 3, url: 'https://example.com/two' }
+      ])
+    end
+
+    it 'rejects an empty URL list' do
+      expect { described_class.parse(" \n\r\n") }
+          .to raise_error(ArgumentError, 'URL を入力してください．')
+    end
+
+    it 'rejects more than 100 non-empty rows' do
+      source = 101.times.map { |index| "https://example.com/#{ index }" }.join("\n")
+
+      expect { described_class.parse(source) }
+          .to raise_error(ArgumentError, '取込件数は 100 件までです．')
+    end
+
+    it 'includes the original line number in an oversized URL error' do
+      source = "\n#{ 'a' * (described_class::MAX_URL_BYTES + 1) }"
+
+      expect { described_class.parse(source) }
+          .to raise_error(ArgumentError, '2 行目: URL が長すぎます．')
+    end
+
+    it 'rejects an oversized request before parsing rows' do
+      source = 'a' * (described_class::MAX_BYTES + 1)
+
+      expect { described_class.parse(source) }
+          .to raise_error(ArgumentError, '入力が大きすぎます．')
+    end
+  end
+end

backend/spec/services/post_metadata_fetcher_spec.rb

@@ -0,0 +1,64 @@
+require 'rails_helper'
+
+RSpec.describe PostMetadataFetcher do
+  Response = Struct.new(:body)
+
+  def fetch_with_published_time(value, url: 'https://example.com/video')
+    html = <<~HTML
+      <html><head>
+        <meta property="article:published_time" content="#{ value }">
+      </head></html>
+    HTML
+    uri = URI.parse(url)
+    allow(Preview::UrlSafety).to receive(:validate).with(url).and_return([uri, ['8.8.8.8']])
+    allow(Preview::HttpFetcher).to receive(:fetch).and_return(Response.new(html))
+    allow(Preview::HtmlMetadataExtractor).to receive(:extract)
+        .and_return(title: 'title', image_url: nil)
+
+    described_class.fetch(url)
+  end
+
+  it 'builds ranges explicitly for year, month, and day precision' do
+    year = fetch_with_published_time('2024')
+    month = fetch_with_published_time('2024-02')
+    day = fetch_with_published_time('2024-02-03')
+
+    expect(year).to include(
+      original_created_from: Time.zone.local(2024, 1, 1).iso8601,
+      original_created_before: Time.zone.local(2025, 1, 1).iso8601
+    )
+    expect(month).to include(
+      original_created_from: Time.zone.local(2024, 2, 1).iso8601,
+      original_created_before: Time.zone.local(2024, 3, 1).iso8601
+    )
+    expect(day).to include(
+      original_created_from: Time.zone.local(2024, 2, 3).iso8601,
+      original_created_before: Time.zone.local(2024, 2, 4).iso8601
+    )
+  end
+
+  it 'preserves an input offset and fractional-second precision' do
+    result = fetch_with_published_time('2024-02-03T12:34:56.123+02:30')
+
+    expect(Time.iso8601(result.fetch(:original_created_from)))
+        .to eq(Time.iso8601('2024-02-03T12:34:56.123+02:30'))
+    expect(
+      Time.iso8601(result.fetch(:original_created_before)) -
+      Time.iso8601(result.fetch(:original_created_from))
+    ).to eq(0.001)
+    expect(result.fetch(:original_created_from)).to include('.123000000')
+  end
+
+  it 'adds platform tags for known video URLs' do
+    result = fetch_with_published_time('2024', url: 'https://youtu.be/abc123')
+
+    expect(result.fetch(:tags)).to eq('動画 YouTube')
+  end
+
+  it 'returns nil dates for an invalid timestamp' do
+    result = fetch_with_published_time('not-a-time')
+
+    expect(result.fetch(:original_created_from)).to be_nil
+    expect(result.fetch(:original_created_before)).to be_nil
+  end
+end

backend/spec/services/post_url_normaliser_spec.rb

@@ -0,0 +1,27 @@
+require 'rails_helper'
+
+RSpec.describe PostUrlNormaliser do
+  before do
+    PostUrlSanitisationRule.unscoped.delete_all
+  end
+
+  describe '.normalise' do
+    it 'normalises the host and trailing slash before applying sanitisation rules' do
+      PostUrlSanitisationRule.create!(
+        priority: 10,
+        source_pattern: '\\Ahttps://example\\.com/source\\z',
+        replacement: 'https://example.com/canonical'
+      )
+
+      result = described_class.normalise('  https://EXAMPLE.com/source/  ')
+
+      expect(result).to eq('https://example.com/canonical')
+    end
+
+    it 'returns nil for unsupported or malformed URLs' do
+      expect(described_class.normalise('ftp://example.com/file')).to be_nil
+      expect(described_class.normalise('https://[')).to be_nil
+      expect(described_class.normalise('https:/path')).to be_nil
+    end
+  end
+end