API の処理に時間がかかりすぎる問題 (#442) (#449)

Reviewed-on: #449
Co-authored-by: miteruzo <miteruzo@naver.com>

backend/app/controllers/post_versions_controller.rb

@@ -14,8 +14,9 @@ class PostVersionsController < ApplicationController
 
     offset = (page - 1) * limit
 
-    q = HistoryAccess.visible(version_scope)
+    q = HistoryAccess.visible(PostVersion.all)
     q = q.where('post_versions.post_id = ?', post_id) if post_id
+    q = q.with_previous_version if tag_id || external_tag_id
     if external_tag_id || (tag_id && !(Tag.exists?(id: tag_id)))
       q = q.where('JSON_CONTAINS(post_versions.tags_json,' +
                   "JSON_OBJECT('external_tag_id', #{ external_tag_id || tag_id })) " +
@@ -28,9 +29,12 @@ class PostVersionsController < ApplicationController
 
     count = q.except(:select, :order, :limit, :offset).count
 
-    versions = q.order(Arel.sql('post_versions.created_at DESC, post_versions.id DESC'))
+    ids = q.order(Arel.sql('post_versions.created_at DESC, post_versions.id DESC'))
                 .limit(limit)
                 .offset(offset)
+                .pluck('post_versions.id')
+    versions = version_scope.merge(q).where(id: ids)
+        .order(Arel.sql('post_versions.created_at DESC, post_versions.id DESC'))
 
     render json: { versions: serialise_versions(versions, locale.language_code), count:, population_count: HistoryAccess.population_count(PostVersion) }
   end

backend/app/controllers/posts_controller.rb

@@ -37,20 +37,10 @@ class PostsController < ApplicationController
 
     offset = (page - 1) * limit
 
-    pt_max_sql =
-        HistoryAccess.visible(PostVersion.all)
-        .select('post_id, MAX(created_at) AS max_updated_at')
-        .group('post_id')
-        .to_sql
-
-    updated_at_all_sql =
-        'GREATEST(posts.updated_at,' +
-        'COALESCE(pt_max.max_updated_at, posts.updated_at))'
+    needs_history_join = order[0] == 'updated_at' || updated_between.any?(&:present?)
 
     q =
         filtered_posts(locale)
-        .joins("LEFT JOIN (#{ pt_max_sql }) pt_max ON pt_max.post_id = posts.id")
-        .reselect('posts.*', Arel.sql("#{ updated_at_all_sql } AS updated_at_all"))
         .preload(:external_tags, :uploaded_user, :parents, :children,
                  post_tags: [:sections, { tag: [:deerjikists, :materials,
                                                 :tag_names, :tag_descriptions] }])
@@ -66,12 +56,25 @@ class PostsController < ApplicationController
     end
     q = q.where('posts.created_at >= ?', created_between[0]) if created_between[0]
     q = q.where('posts.created_at <= ?', created_between[1]) if created_between[1]
+    count_scope = q
+    if needs_history_join
+      history_times = HistoryAccess.visible(PostVersion.all)
+          .select('post_id, MAX(created_at) AS latest_at').group(:post_id)
+      q = q.joins("LEFT JOIN (#{ history_times.to_sql }) history_times " +
+                  'ON history_times.post_id = posts.id')
+      updated_at_all_sql = 'GREATEST(posts.updated_at, ' +
+          'COALESCE(history_times.latest_at, posts.updated_at))'
+    end
     if updated_between[0]
       q = q.where("#{ updated_at_all_sql } >= ?", updated_between[0])
     end
     if updated_between[1]
       q = q.where("#{ updated_at_all_sql } <= ?", updated_between[1])
     end
+    count_scope = q if updated_between.any?(&:present?)
+    if needs_history_join
+      q = q.select('posts.*', "#{ updated_at_all_sql } AS updated_at")
+    end
 
     sort_sql =
         case order[0]
@@ -88,12 +91,20 @@ class PostsController < ApplicationController
              .limit(limit)
              .offset(offset)
              .to_a
+    latest_version_times =
+        if needs_history_join
+          { }
+        else
+          HistoryAccess.visible(PostVersion.where(post_id: posts.map(&:id)))
+              .group(:post_id).maximum(:created_at)
+        end
 
-    q = q.except(:select, :order)
+    count_scope = count_scope.except(:select, :order)
 
     render json: { posts: posts.map { |post|
+      updated_at = [post.updated_at, latest_version_times[post.id]].compact.max
       PostRepr.base(post, language_code: locale.language_code)
-              .merge(updated_at: post.updated_at_all).tap do |json|
+          .merge('updated_at' => updated_at).tap do |json|
         json['thumbnail'] =
             if post.thumbnail.attached?
               rails_storage_proxy_url(post.thumbnail, only_path: false)
@@ -101,7 +112,7 @@ class PostsController < ApplicationController
               nil
             end
       end
-    }, count: q.group_values.present? ? q.count.size : q.count }
+    }, count: count_scope.group_values.present? ? count_scope.count.size : count_scope.count }
   end
 
   def random

backend/app/models/history_access.rb

@@ -59,20 +59,21 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
   end
 
   def self.json_reference(table, key, document)
-    "NOT EXISTS (SELECT 1 FROM JSON_TABLE(COALESCE(#{ document }, JSON_ARRAY()), " +
+    # A scalar lookup avoids MySQL's semijoin rewrite losing the JSON row correlation.
+    "(SELECT 1 FROM JSON_TABLE(COALESCE(#{ document }, JSON_ARRAY()), " +
         "'$[*]' COLUMNS (reference_id BIGINT PATH '$' NULL ON EMPTY)) reference_entry " +
         "JOIN (#{ discarded_keys(table, key) }) blocked " +
-        "ON reference_entry.reference_id = blocked.#{ key })"
+        "ON reference_entry.reference_id = blocked.#{ key } LIMIT 1) IS NULL"
   end
 
   def self.list_reference(table, key, document)
     # Preserve FIND_IN_SET's exact token semantics, including empty tokens.
     array = "CONCAT('[', REPLACE(JSON_QUOTE(REPLACE(COALESCE(#{ document }, ''), " +
         "' ', ',')), ',', '\",\"'), ']')"
-    "NOT EXISTS (SELECT 1 FROM JSON_TABLE(#{ array }, '$[*]' " +
+    "(SELECT 1 FROM JSON_TABLE(#{ array }, '$[*]' " +
         "COLUMNS (reference_id VARCHAR(255) PATH '$')) reference_entry " +
         "JOIN (#{ discarded_keys(table, key) }) blocked " +
-        "ON reference_entry.reference_id = CAST(blocked.#{ key } AS CHAR))"
+        "ON reference_entry.reference_id = CAST(blocked.#{ key } AS CHAR) LIMIT 1) IS NULL"
   end
 
   def self.snapshot_reference(model, row, external: false)
@@ -91,8 +92,8 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
       match += ' AND reference_version.version_no = tag_reference.version_no'
       latest = ''
     end
-    "NOT EXISTS (SELECT 1 FROM #{ source } JOIN #{ table } reference_version ON #{ match } " +
-        "WHERE NOT (#{ sql(model, 'reference_version') }) #{ latest })"
+    "(SELECT 1 FROM #{ source } JOIN #{ table } reference_version ON #{ match } " +
+        "WHERE NOT (#{ sql(model, 'reference_version') }) #{ latest } LIMIT 1) IS NULL"
   end
 
   def self.reference_sql(table, row)
@@ -109,11 +110,15 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
       # conservatively; never expose a private linked name through an external tag.
       names = "JSON_MERGE_PRESERVE(JSON_ARRAY(private_tag.name), " +
           "COALESCE(JSON_EXTRACT(private_tag.names, '$**.name'), JSON_ARRAY()))"
-      ["NOT EXISTS (SELECT 1 FROM tag_versions private_tag " +
+      # DISTINCT prevents merging this independent name set into the correlated scan.
+      # Keep byte-distinct names even when the column collation considers them equal.
+      private_names = 'SELECT DISTINCT private_name.name, ' +
+          'CAST(private_name.name AS BINARY) AS name_bytes FROM tag_versions private_tag ' +
           "JOIN (#{ discarded_keys('tag_versions', 'tag_id') }) blocked " +
-          "ON blocked.tag_id = private_tag.tag_id " +
+          'ON blocked.tag_id = private_tag.tag_id ' +
           "JOIN JSON_TABLE(#{ names }, '$[*]' COLUMNS (name TEXT PATH '$')) private_name " +
-          "WHERE private_name.name <> '' AND " +
+          "WHERE private_name.name <> ''"
+      ["NOT EXISTS (SELECT 1 FROM (#{ private_names }) private_name WHERE " +
           "LOCATE(CONCAT(' ', private_name.name, ' '), " +
           "CONCAT(' ', COALESCE(#{ row }.linked_tags, ''), ' ')) > 0)"]
     when 'tag_versions'