diff --git a/backend/Gemfile b/backend/Gemfile index 7b44641..ba2bbb1 100644 --- a/backend/Gemfile +++ b/backend/Gemfile @@ -71,3 +71,5 @@ gem "rspec-rails", "~> 8.0", :groups => [:development, :test] gem 'aws-sdk-s3', require: false gem 'rails-i18n', '~> 8.0.0' + +gem "json", "~> 2.19.9" diff --git a/backend/Gemfile.lock b/backend/Gemfile.lock index 555f867..c130664 100644 --- a/backend/Gemfile.lock +++ b/backend/Gemfile.lock @@ -1,29 +1,29 @@ GEM remote: https://rubygems.org/ specs: - actioncable (8.0.2) - actionpack (= 8.0.2) - activesupport (= 8.0.2) + actioncable (8.0.5.1) + actionpack (= 8.0.5.1) + activesupport (= 8.0.5.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (8.0.2) - actionpack (= 8.0.2) - activejob (= 8.0.2) - activerecord (= 8.0.2) - activestorage (= 8.0.2) - activesupport (= 8.0.2) + actionmailbox (8.0.5.1) + actionpack (= 8.0.5.1) + activejob (= 8.0.5.1) + activerecord (= 8.0.5.1) + activestorage (= 8.0.5.1) + activesupport (= 8.0.5.1) mail (>= 2.8.0) - actionmailer (8.0.2) - actionpack (= 8.0.2) - actionview (= 8.0.2) - activejob (= 8.0.2) - activesupport (= 8.0.2) + actionmailer (8.0.5.1) + actionpack (= 8.0.5.1) + actionview (= 8.0.5.1) + activejob (= 8.0.5.1) + activesupport (= 8.0.5.1) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (8.0.2) - actionview (= 8.0.2) - activesupport (= 8.0.2) + actionpack (8.0.5.1) + actionview (= 8.0.5.1) + activesupport (= 8.0.5.1) nokogiri (>= 1.8.5) rack (>= 2.2.4) rack-session (>= 1.0.1) @@ -31,35 +31,35 @@ GEM rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (8.0.2) - actionpack (= 8.0.2) - activerecord (= 8.0.2) - activestorage (= 8.0.2) - activesupport (= 8.0.2) + actiontext (8.0.5.1) + actionpack (= 8.0.5.1) + activerecord (= 8.0.5.1) + activestorage (= 8.0.5.1) + activesupport (= 8.0.5.1) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (8.0.2) - activesupport (= 8.0.2) + actionview (8.0.5.1) + activesupport (= 8.0.5.1) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (8.0.2) - activesupport (= 8.0.2) + activejob (8.0.5.1) + activesupport (= 8.0.5.1) globalid (>= 0.3.6) - activemodel (8.0.2) - activesupport (= 8.0.2) - activerecord (8.0.2) - activemodel (= 8.0.2) - activesupport (= 8.0.2) + activemodel (8.0.5.1) + activesupport (= 8.0.5.1) + activerecord (8.0.5.1) + activemodel (= 8.0.5.1) + activesupport (= 8.0.5.1) timeout (>= 0.4.0) - activestorage (8.0.2) - actionpack (= 8.0.2) - activejob (= 8.0.2) - activerecord (= 8.0.2) - activesupport (= 8.0.2) + activestorage (8.0.5.1) + actionpack (= 8.0.5.1) + activejob (= 8.0.5.1) + activerecord (= 8.0.5.1) + activesupport (= 8.0.5.1) marcel (~> 1.0) - activesupport (8.0.2) + activesupport (8.0.5.1) base64 benchmark (>= 0.3) bigdecimal @@ -92,22 +92,22 @@ GEM aws-sigv4 (~> 1.5) aws-sigv4 (1.12.1) aws-eventstream (~> 1, >= 1.0.2) - base64 (0.2.0) + base64 (0.3.0) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) bcrypt_pbkdf (1.1.1-x86_64-darwin) - benchmark (0.4.0) - bigdecimal (3.1.9) + benchmark (0.5.0) + bigdecimal (4.1.3) bootsnap (1.18.6) msgpack (~> 1.2) brakeman (7.0.2) racc builder (3.3.0) chronic (0.10.2) - concurrent-ruby (1.3.5) - connection_pool (2.5.3) - crass (1.0.6) - date (3.4.1) + concurrent-ruby (1.3.8) + connection_pool (3.0.2) + crass (1.0.7) + date (3.6.0) diff-lcs (1.6.2) discard (1.4.0) activerecord (>= 4.2, < 9.0) @@ -115,8 +115,9 @@ GEM dotenv-rails (3.1.8) dotenv (= 3.1.8) railties (>= 6.1) - drb (2.2.1) + drb (2.2.3) ed25519 (1.4.0) + erb (6.0.7) erubi (1.13.1) factory_bot (6.5.6) activesupport (>= 6.1.0) @@ -134,7 +135,7 @@ GEM gemojione (4.3.3) json github-markup (4.0.2) - globalid (1.2.1) + globalid (1.4.0) activesupport (>= 6.1) gollum (6.1.0) gemojione (~> 4.1) @@ -166,19 +167,20 @@ GEM gollum-rugged_adapter (3.0) mime-types (~> 3.4) rugged (~> 1.5) - i18n (1.14.7) + i18n (1.15.2) concurrent-ruby (~> 1.0) image_processing (1.14.0) mini_magick (>= 4.9.5, < 6) ruby-vips (>= 2.0.17, < 3) - io-console (0.8.0) - irb (1.15.2) + io-console (0.9.4) + irb (1.18.0) pp (>= 0.6.0) + prism (>= 1.3.0) rdoc (>= 4.0.0) reline (>= 0.4.2) jmespath (1.6.2) - json (2.12.0) - jwt (2.10.1) + json (2.19.9) + jwt (3.3.0) base64 kamal (2.6.1) activesupport (>= 7.0) @@ -198,15 +200,16 @@ GEM language_server-protocol (3.17.0.5) lint_roller (1.1.0) logger (1.7.0) - loofah (2.24.1) + loofah (2.25.2) crass (~> 1.0.2) nokogiri (>= 1.12.0) - mail (2.8.1) + mail (2.9.1) + logger mini_mime (>= 0.1.1) net-imap net-pop net-smtp - marcel (1.0.4) + marcel (1.2.1) mime-types (3.7.0) logger mime-types-data (~> 3.2025, >= 3.2025.0507) @@ -215,45 +218,46 @@ GEM benchmark logger mini_mime (1.1.5) - minitest (5.25.5) - msgpack (1.8.0) - multi_json (1.15.0) + minitest (6.0.6) + drb (~> 2.0) + prism (~> 1.5) + msgpack (1.8.5) + multi_json (1.21.2) mustache (1.1.1) mustache-sinatra (2.0.0) mustache (~> 1.0) - mustermann (3.0.3) - ruby2_keywords (~> 0.0.1) + mustermann (3.1.1) mysql2 (0.5.6) - net-imap (0.5.8) + net-imap (0.6.8) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.2) + net-protocol (0.4.0) timeout net-scp (4.1.0) net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) - net-smtp (0.5.1) + net-smtp (0.5.2) net-protocol net-ssh (7.3.0) - nio4r (2.7.4) - nokogiri (1.18.8-aarch64-linux-gnu) + nio4r (2.7.5) + nokogiri (1.19.4-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.8-aarch64-linux-musl) + nokogiri (1.19.4-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.18.8-arm-linux-gnu) + nokogiri (1.19.4-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.18.8-arm-linux-musl) + nokogiri (1.19.4-arm-linux-musl) racc (~> 1.4) - nokogiri (1.18.8-arm64-darwin) + nokogiri (1.19.4-arm64-darwin) racc (~> 1.4) - nokogiri (1.18.8-x86_64-darwin) + nokogiri (1.19.4-x86_64-darwin) racc (~> 1.4) - nokogiri (1.18.8-x86_64-linux-gnu) + nokogiri (1.19.4-x86_64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.8-x86_64-linux-musl) + nokogiri (1.19.4-x86_64-linux-musl) racc (~> 1.4) octicons (19.15.2) ostruct (0.6.1) @@ -261,70 +265,73 @@ GEM parser (3.3.8.0) ast (~> 2.4.1) racc - pp (0.6.2) + pp (0.6.4) prettyprint - prettyprint (0.2.0) - prism (1.4.0) - psych (5.2.6) + prettyprint (0.2.1) + prism (1.9.0) + psych (5.5.0) date stringio puma (6.6.0) nio4r (~> 2.0) racc (1.8.1) - rack (3.1.14) + rack (3.2.7) rack-cors (2.0.2) rack (>= 2.0.0) - rack-protection (4.1.1) + rack-protection (4.2.1) base64 (>= 0.1.0) logger (>= 1.6.0) rack (>= 3.0.0, < 4) - rack-session (2.1.1) + rack-session (2.1.2) base64 (>= 0.1.0) rack (>= 3.0.0) - rack-test (2.2.0) + rack-test (2.3.0) rack (>= 1.3) - rackup (2.2.1) + rackup (2.3.1) rack (>= 3) - rails (8.0.2) - actioncable (= 8.0.2) - actionmailbox (= 8.0.2) - actionmailer (= 8.0.2) - actionpack (= 8.0.2) - actiontext (= 8.0.2) - actionview (= 8.0.2) - activejob (= 8.0.2) - activemodel (= 8.0.2) - activerecord (= 8.0.2) - activestorage (= 8.0.2) - activesupport (= 8.0.2) + rails (8.0.5.1) + actioncable (= 8.0.5.1) + actionmailbox (= 8.0.5.1) + actionmailer (= 8.0.5.1) + actionpack (= 8.0.5.1) + actiontext (= 8.0.5.1) + actionview (= 8.0.5.1) + activejob (= 8.0.5.1) + activemodel (= 8.0.5.1) + activerecord (= 8.0.5.1) + activestorage (= 8.0.5.1) + activesupport (= 8.0.5.1) bundler (>= 1.15.0) - railties (= 8.0.2) - rails-dom-testing (2.2.0) + railties (= 8.0.5.1) + rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.2) - loofah (~> 2.21) + rails-html-sanitizer (1.7.1) + loofah (~> 2.25, >= 2.25.2) nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) rails-i18n (8.0.2) i18n (>= 0.7, < 2) railties (>= 8.0.0, < 9) - railties (8.0.2) - actionpack (= 8.0.2) - activesupport (= 8.0.2) + railties (8.0.5.1) + actionpack (= 8.0.5.1) + activesupport (= 8.0.5.1) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) + tsort (>= 0.2) zeitwerk (~> 2.6) rainbow (3.1.1) - rake (13.2.1) - rdoc (6.13.1) + rake (13.4.2) + rdoc (6.17.0) + erb psych (>= 4.0.0) - regexp_parser (2.10.0) - reline (0.6.1) + tsort + regexp_parser (2.13.1) + reline (0.7.0) io-console (~> 0.5) - rexml (3.4.1) + rexml (3.4.4) rouge (3.30.0) rspec-core (3.13.6) rspec-support (~> 3.13.0) @@ -345,20 +352,20 @@ GEM rspec-support (3.13.6) rss (0.3.1) rexml - rubocop (1.75.6) - json (~> 2.3) + rubocop (1.92.0) + json (>= 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) - parallel (~> 1.10) + parallel (>= 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.44.0, < 2.0) + regexp_parser (>= 2.11.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.44.1) + rubocop-ast (1.50.0) parser (>= 3.3.7.2) - prism (~> 1.4) + prism (~> 1.7) rubocop-performance (1.25.0) lint_roller (~> 1.1) rubocop (>= 1.75.0, < 2.0) @@ -377,24 +384,23 @@ GEM ruby-vips (2.2.3) ffi (~> 1.12) logger - ruby2_keywords (0.0.5) rugged (1.9.0) securerandom (0.4.1) - sinatra (4.1.1) + sinatra (4.2.1) logger (>= 1.6.0) mustermann (~> 3.0) rack (>= 3.0.0, < 4) - rack-protection (= 4.1.1) + rack-protection (= 4.2.1) rack-session (>= 2.0.0, < 3) tilt (~> 2.0) - sinatra-contrib (4.1.1) + sinatra-contrib (4.2.1) multi_json (>= 0.0.2) mustermann (~> 3.0) - rack-protection (= 4.1.1) - sinatra (= 4.1.1) + rack-protection (= 4.2.1) + sinatra (= 4.2.1) tilt (~> 2.0) - sprockets (4.2.2) - concurrent-ruby (~> 1.0) + sprockets (4.4.1) + concurrent-ruby (~> 1.1) logger rack (>= 2.2.4, < 4) sprockets-helpers (1.4.0) @@ -403,14 +409,14 @@ GEM actionpack (>= 6.1) activesupport (>= 6.1) sprockets (>= 3.0.0) - sqlite3 (2.6.0-aarch64-linux-gnu) - sqlite3 (2.6.0-aarch64-linux-musl) - sqlite3 (2.6.0-arm-linux-gnu) - sqlite3 (2.6.0-arm-linux-musl) - sqlite3 (2.6.0-arm64-darwin) - sqlite3 (2.6.0-x86_64-darwin) - sqlite3 (2.6.0-x86_64-linux-gnu) - sqlite3 (2.6.0-x86_64-linux-musl) + sqlite3 (2.10.1-aarch64-linux-gnu) + sqlite3 (2.10.1-aarch64-linux-musl) + sqlite3 (2.10.1-arm-linux-gnu) + sqlite3 (2.10.1-arm-linux-musl) + sqlite3 (2.10.1-arm64-darwin) + sqlite3 (2.10.1-x86_64-darwin) + sqlite3 (2.10.1-x86_64-linux-gnu) + sqlite3 (2.10.1-x86_64-linux-musl) sshkit (1.24.0) base64 logger @@ -418,18 +424,19 @@ GEM net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) ostruct - stringio (3.1.7) + stringio (3.2.0) therubyrhino (2.1.2) therubyrhino_jar (>= 1.7.4, < 1.7.9) therubyrhino_jar (1.7.8) - thor (1.3.2) + thor (1.5.0) thruster (0.1.13) thruster (0.1.13-aarch64-linux) thruster (0.1.13-arm64-darwin) thruster (0.1.13-x86_64-darwin) thruster (0.1.13-x86_64-linux) - tilt (2.6.0) - timeout (0.4.3) + tilt (2.9.0) + timeout (0.6.1) + tsort (0.2.0) twitter-text (1.14.7) unf (~> 0.1.0) tzinfo (2.0.6) @@ -440,16 +447,16 @@ GEM unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) - uri (1.0.3) + uri (1.1.2) useragent (0.16.11) webrick (1.9.1) - websocket-driver (0.7.7) + websocket-driver (0.8.2) base64 websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) whenever (1.0.0) chronic (>= 0.6.3) - zeitwerk (2.7.2) + zeitwerk (2.8.3) PLATFORMS aarch64-linux @@ -473,6 +480,7 @@ DEPENDENCIES factory_bot_rails gollum image_processing (~> 1.14) + json (~> 2.19.9) jwt kamal mysql2 (~> 0.5.6) diff --git a/backend/app/controllers/post_versions_controller.rb b/backend/app/controllers/post_versions_controller.rb index 51f6f3f..cf07119 100644 --- a/backend/app/controllers/post_versions_controller.rb +++ b/backend/app/controllers/post_versions_controller.rb @@ -14,8 +14,9 @@ class PostVersionsController < ApplicationController offset = (page - 1) * limit - q = HistoryAccess.visible(version_scope) + q = HistoryAccess.visible(PostVersion.all) q = q.where('post_versions.post_id = ?', post_id) if post_id + q = q.with_previous_version if tag_id || external_tag_id if external_tag_id || (tag_id && !(Tag.exists?(id: tag_id))) q = q.where('JSON_CONTAINS(post_versions.tags_json,' + "JSON_OBJECT('external_tag_id', #{ external_tag_id || tag_id })) " + @@ -28,9 +29,12 @@ class PostVersionsController < ApplicationController count = q.except(:select, :order, :limit, :offset).count - versions = q.order(Arel.sql('post_versions.created_at DESC, post_versions.id DESC')) + ids = q.order(Arel.sql('post_versions.created_at DESC, post_versions.id DESC')) .limit(limit) .offset(offset) + .pluck('post_versions.id') + versions = version_scope.merge(q).where(id: ids) + .order(Arel.sql('post_versions.created_at DESC, post_versions.id DESC')) render json: { versions: serialise_versions(versions, locale.language_code), count:, population_count: HistoryAccess.population_count(PostVersion) } end diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 5665fa7..ba15e99 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -37,20 +37,10 @@ class PostsController < ApplicationController offset = (page - 1) * limit - pt_max_sql = - HistoryAccess.visible(PostVersion.all) - .select('post_id, MAX(created_at) AS max_updated_at') - .group('post_id') - .to_sql - - updated_at_all_sql = - 'GREATEST(posts.updated_at,' + - 'COALESCE(pt_max.max_updated_at, posts.updated_at))' + needs_history_join = order[0] == 'updated_at' || updated_between.any?(&:present?) q = filtered_posts(locale) - .joins("LEFT JOIN (#{ pt_max_sql }) pt_max ON pt_max.post_id = posts.id") - .reselect('posts.*', Arel.sql("#{ updated_at_all_sql } AS updated_at_all")) .preload(:external_tags, :uploaded_user, :parents, :children, post_tags: [:sections, { tag: [:deerjikists, :materials, :tag_names, :tag_descriptions] }]) @@ -66,12 +56,25 @@ class PostsController < ApplicationController end q = q.where('posts.created_at >= ?', created_between[0]) if created_between[0] q = q.where('posts.created_at <= ?', created_between[1]) if created_between[1] + count_scope = q + if needs_history_join + history_times = HistoryAccess.visible(PostVersion.all) + .select('post_id, MAX(created_at) AS latest_at').group(:post_id) + q = q.joins("LEFT JOIN (#{ history_times.to_sql }) history_times " + + 'ON history_times.post_id = posts.id') + updated_at_all_sql = 'GREATEST(posts.updated_at, ' + + 'COALESCE(history_times.latest_at, posts.updated_at))' + end if updated_between[0] q = q.where("#{ updated_at_all_sql } >= ?", updated_between[0]) end if updated_between[1] q = q.where("#{ updated_at_all_sql } <= ?", updated_between[1]) end + count_scope = q if updated_between.any?(&:present?) + if needs_history_join + q = q.select('posts.*', "#{ updated_at_all_sql } AS updated_at") + end sort_sql = case order[0] @@ -88,12 +91,20 @@ class PostsController < ApplicationController .limit(limit) .offset(offset) .to_a + latest_version_times = + if needs_history_join + { } + else + HistoryAccess.visible(PostVersion.where(post_id: posts.map(&:id))) + .group(:post_id).maximum(:created_at) + end - q = q.except(:select, :order) + count_scope = count_scope.except(:select, :order) render json: { posts: posts.map { |post| + updated_at = [post.updated_at, latest_version_times[post.id]].compact.max PostRepr.base(post, language_code: locale.language_code) - .merge(updated_at: post.updated_at_all).tap do |json| + .merge('updated_at' => updated_at).tap do |json| json['thumbnail'] = if post.thumbnail.attached? rails_storage_proxy_url(post.thumbnail, only_path: false) @@ -101,7 +112,7 @@ class PostsController < ApplicationController nil end end - }, count: q.group_values.present? ? q.count.size : q.count } + }, count: count_scope.group_values.present? ? count_scope.count.size : count_scope.count } end def random diff --git a/backend/app/models/history_access.rb b/backend/app/models/history_access.rb index 0d9f92a..cb3de89 100644 --- a/backend/app/models/history_access.rb +++ b/backend/app/models/history_access.rb @@ -59,20 +59,21 @@ class HistoryAccess < ActiveSupport::CurrentAttributes end def self.json_reference(table, key, document) - "NOT EXISTS (SELECT 1 FROM JSON_TABLE(COALESCE(#{ document }, JSON_ARRAY()), " + + # A scalar lookup avoids MySQL's semijoin rewrite losing the JSON row correlation. + "(SELECT 1 FROM JSON_TABLE(COALESCE(#{ document }, JSON_ARRAY()), " + "'$[*]' COLUMNS (reference_id BIGINT PATH '$' NULL ON EMPTY)) reference_entry " + "JOIN (#{ discarded_keys(table, key) }) blocked " + - "ON reference_entry.reference_id = blocked.#{ key })" + "ON reference_entry.reference_id = blocked.#{ key } LIMIT 1) IS NULL" end def self.list_reference(table, key, document) # Preserve FIND_IN_SET's exact token semantics, including empty tokens. array = "CONCAT('[', REPLACE(JSON_QUOTE(REPLACE(COALESCE(#{ document }, ''), " + "' ', ',')), ',', '\",\"'), ']')" - "NOT EXISTS (SELECT 1 FROM JSON_TABLE(#{ array }, '$[*]' " + + "(SELECT 1 FROM JSON_TABLE(#{ array }, '$[*]' " + "COLUMNS (reference_id VARCHAR(255) PATH '$')) reference_entry " + "JOIN (#{ discarded_keys(table, key) }) blocked " + - "ON reference_entry.reference_id = CAST(blocked.#{ key } AS CHAR))" + "ON reference_entry.reference_id = CAST(blocked.#{ key } AS CHAR) LIMIT 1) IS NULL" end def self.snapshot_reference(model, row, external: false) @@ -91,8 +92,8 @@ class HistoryAccess < ActiveSupport::CurrentAttributes match += ' AND reference_version.version_no = tag_reference.version_no' latest = '' end - "NOT EXISTS (SELECT 1 FROM #{ source } JOIN #{ table } reference_version ON #{ match } " + - "WHERE NOT (#{ sql(model, 'reference_version') }) #{ latest })" + "(SELECT 1 FROM #{ source } JOIN #{ table } reference_version ON #{ match } " + + "WHERE NOT (#{ sql(model, 'reference_version') }) #{ latest } LIMIT 1) IS NULL" end def self.reference_sql(table, row) @@ -109,11 +110,15 @@ class HistoryAccess < ActiveSupport::CurrentAttributes # conservatively; never expose a private linked name through an external tag. names = "JSON_MERGE_PRESERVE(JSON_ARRAY(private_tag.name), " + "COALESCE(JSON_EXTRACT(private_tag.names, '$**.name'), JSON_ARRAY()))" - ["NOT EXISTS (SELECT 1 FROM tag_versions private_tag " + + # DISTINCT prevents merging this independent name set into the correlated scan. + # Keep byte-distinct names even when the column collation considers them equal. + private_names = 'SELECT DISTINCT private_name.name, ' + + 'CAST(private_name.name AS BINARY) AS name_bytes FROM tag_versions private_tag ' + "JOIN (#{ discarded_keys('tag_versions', 'tag_id') }) blocked " + - "ON blocked.tag_id = private_tag.tag_id " + + 'ON blocked.tag_id = private_tag.tag_id ' + "JOIN JSON_TABLE(#{ names }, '$[*]' COLUMNS (name TEXT PATH '$')) private_name " + - "WHERE private_name.name <> '' AND " + + "WHERE private_name.name <> ''" + ["NOT EXISTS (SELECT 1 FROM (#{ private_names }) private_name WHERE " + "LOCATE(CONCAT(' ', private_name.name, ' '), " + "CONCAT(' ', COALESCE(#{ row }.linked_tags, ''), ' ')) > 0)"] when 'tag_versions' diff --git a/backend/spec/requests/post_history_privacy_spec.rb b/backend/spec/requests/post_history_privacy_spec.rb new file mode 100644 index 0000000..635fb76 --- /dev/null +++ b/backend/spec/requests/post_history_privacy_spec.rb @@ -0,0 +1,129 @@ +require 'rails_helper' + +RSpec.describe 'Post history privacy', type: :request do + let(:origin) { Time.utc(2020, 1, 1) } + let!(:post_record) do + create(:post, title: 'current', created_at: origin, updated_at: origin) + end + let!(:other_post) do + create(:post, created_at: origin, updated_at: origin + 3.days + 1.hour) + end + let!(:tag) { create(:tag, primary_name: 'private-history-tag') } + let!(:external) { create(:external_tag, name: 'external-history-tag') } + + before do + ['create', 'discard'].each_with_index do |event, index| + TagVersion.create!(tag:, version_no: index + 1, event_type: event, + name: 'private-history-tag', category: 'general', aliases: '', parent_tag_ids: '', + names: { ja: { name: 'private-history-tag', script_code: 'Jpan', + auto_generated: false, aliases: [] } }, + created_at: origin + index.hours) + end + NicoTagVersion.create!(external_tag: external, version_no: 1, event_type: 'create', + name: 'nico:external-history-tag', linked_tags: 'private-history-tag', + created_at: origin) + ['create', 'discard', 'restore', 'update'].each_with_index do |event, index| + snapshots = + if index < 2 + [{ tag_id: tag.id, version_no: 1, name: 'private-history-tag', sections: [] }] + elsif index == 3 + [{ external_tag_id: external.id }] + else + [] + end + PostVersion.create!(post: post_record, version_no: index + 1, event_type: event, + title: "version #{ index + 1 }", url: post_record.url, tags: '', + tags_json: snapshots, parent_post_ids: '', created_at: origin + (index + 1).days) + end + end + + it 'evaluates each reference condition against the restored snapshot alone' do + HistoryAccess.set(user: nil) do + restored = PostVersion.where(post_id: post_record.id, version_no: 3) + HistoryAccess.reference_sql('post_versions', 'post_versions').each do |condition| + expect(restored.where(Arel.sql(condition)).pluck(:version_no)).to eq([3]) + end + expect(HistoryAccess.visible(PostVersion.where(post_id: post_record.id)) + .pluck(:version_no)).to eq([3]) + end + end + + it 'keeps every version through a later discard private after another restore' do + ['discard', 'restore'].each_with_index do |event, index| + PostVersion.create!(post: post_record, version_no: index + 5, event_type: event, + title: 'later snapshot', url: post_record.url, tags: '', tags_json: [], + parent_post_ids: '', created_at: origin + (index + 5).days) + end + + get '/posts/versions', params: { post: post_record.id } + + expect(response).to have_http_status(:ok) + expect(response.parsed_body.fetch('versions').pluck('version_no')).to eq([6]) + expect(response.parsed_body.fetch('count')).to eq(1) + expect(response.parsed_body.fetch('versions').first.fetch('title').fetch('prev')).to be_nil + end + + ['anonymous', 'member', 'admin'].each do |role| + context "as #{ role }" do + before do + sign_in_as(create(:user, role.to_sym)) unless role == 'anonymous' + end + + it 'uses only authorised history timestamps for display' do + get '/posts' + expect(response).to have_http_status(:ok) + row = response.parsed_body.fetch('posts').find { _1.fetch('id') == post_record.id } + expected = origin + (role == 'admin' ? 4 : 3).days + expect(Time.zone.parse(row.fetch('updated_at'))).to eq(expected) + end + + it 'uses the same visibility for date bounds and count' do + get '/posts', params: { updated_from: (origin + 4.days).iso8601, + updated_to: (origin + 4.days).iso8601 } + expect(response).to have_http_status(:ok) + ids = response.parsed_body.fetch('posts').pluck('id') + expect(ids).to eq(role == 'admin' ? [post_record.id] : []) + expect(response.parsed_body.fetch('count')).to eq(ids.length) + end + + it 'sorts using authorised timestamps in both directions' do + ['asc', 'desc'].each do |direction| + get '/posts', params: { order: "updated_at:#{ direction }" } + expect(response).to have_http_status(:ok) + expected = role == 'admin' ? [other_post.id, post_record.id] : + [post_record.id, other_post.id] + expected.reverse! if direction == 'desc' + expect(response.parsed_body.fetch('posts').pluck('id')).to eq(expected) + end + end + + it 'applies authorised timestamp ordering before pagination' do + get '/posts', params: { order: 'updated_at:desc', limit: 1 } + + expect(response).to have_http_status(:ok) + expected = role == 'admin' ? post_record.id : other_post.id + expect(response.parsed_body.fetch('posts').pluck('id')).to eq([expected]) + expect(response.parsed_body.fetch('count')).to eq(2) + end + + it 'counts only visible versions and masks a private previous version' do + get '/posts/versions', params: { post: post_record.id } + expect(response).to have_http_status(:ok) + body = response.parsed_body + expected = role == 'admin' ? [4, 3, 2, 1] : [3] + expect(body.fetch('count')).to eq(expected.length) + expect(body.fetch('versions').pluck('version_no')).to eq(expected) + restored = body.fetch('versions').find { _1.fetch('version_no') == 3 } + expect(restored.fetch('title').fetch('prev')).to eq(role == 'admin' ? 'version 2' : nil) + end + + it 'uses only an authorised previous version for tag filtering' do + get '/posts/versions', params: { post: post_record.id, tag: tag.id } + expect(response).to have_http_status(:ok) + expected = role == 'admin' ? [3, 2, 1] : [] + expect(response.parsed_body.fetch('versions').pluck('version_no')).to eq(expected) + expect(response.parsed_body.fetch('count')).to eq(expected.length) + end + end + end +end diff --git a/backend/spec/requests/posts_spec.rb b/backend/spec/requests/posts_spec.rb index f0d0ab7..aa5f7c1 100644 --- a/backend/spec/requests/posts_spec.rb +++ b/backend/spec/requests/posts_spec.rb @@ -629,6 +629,37 @@ RSpec.describe 'Posts API', type: :request do create(:post_tag, post: plain_post, tag:, created_at: t3) end + it 'displays history time after pagination without joining history to posts or count' do + queries = [] + subscriber = ->(event) { queries << event.payload.fetch(:sql) } + ActiveSupport::Notifications.subscribed(subscriber, 'sql.active_record') do + get '/posts', params: { url: '/version-time/', order: 'url:asc', limit: 1 } + end + + expect(response).to have_http_status(:ok) + expect(json.fetch('posts').pluck('id')).to eq([history_post.id]) + expect(Time.zone.parse(json.fetch('posts').first.fetch('updated_at'))).to eq(t2) + expect(json.fetch('count')).to eq(3) + post_queries = queries.select { _1.include?('FROM `posts`') } + expect(post_queries).not_to be_empty + expect(post_queries.grep(/post_versions/)).to be_empty + end + + it 'omits the history aggregate from count when only sorting by update time' do + queries = [] + subscriber = ->(event) { queries << event.payload.fetch(:sql) } + ActiveSupport::Notifications.subscribed(subscriber, 'sql.active_record') do + get '/posts', params: { url: '/version-time/', order: 'updated_at:desc', limit: 1 } + end + + expect(response).to have_http_status(:ok) + expect(json.fetch('posts').pluck('id')).to eq([newer_post.id]) + expect(json.fetch('count')).to eq(3) + count_queries = queries.select { _1.include?('FROM `posts`') && _1.match?(/COUNT\(/i) } + expect(count_queries).not_to be_empty + expect(count_queries.grep(/post_versions/)).to be_empty + end + ['asc', 'desc'].each do |direction| it "sorts by the later of post update and latest version time (#{ direction })" do get '/posts', params: { url: '/version-time/', order: "updated_at:#{ direction }" } @@ -658,6 +689,16 @@ RSpec.describe 'Posts API', type: :request do expect(json.fetch('posts').map { |item| item.fetch('id') }).to eq([history_post.id]) expect(json.fetch('count')).to eq(1) end + + it 'counts every timestamp match before pagination' do + get '/posts', params: { url: '/version-time/', order: 'updated_at:asc', + updated_from: t1.iso8601, updated_to: t3.iso8601, + limit: 1, page: 2 } + + expect(response).to have_http_status(:ok) + expect(json.fetch('posts').pluck('id')).to eq([history_post.id]) + expect(json.fetch('count')).to eq(3) + end end context 'when original_created_from/original_created_to are provided' do diff --git a/backend/spec/services/counts_export_spec.rb b/backend/spec/services/counts_export_spec.rb index de205d1..94b82c1 100644 --- a/backend/spec/services/counts_export_spec.rb +++ b/backend/spec/services/counts_export_spec.rb @@ -221,9 +221,15 @@ RSpec.describe CountsExportDownload::ResponseBody do body = described_class.new(source) { |state| states << state; true } response = ActionDispatch::Response.new response.body = body - deferred = response.stream.to_ary + + _, _, rack_body = response.to_a + expect(states).to eq([]) - expect(deferred.to_a.join).to eq(payload) + + chunks = [] + rack_body.each { |chunk| chunks << chunk } + + expect(chunks.join).to eq(payload) expect(states.count('started')).to eq(1) end end