Merge branch 'master' of https://github.com/otwcode/otwarchive into A…

…O3-5937

.github/workflows/reviewdog.yml

@@ -11,6 +11,8 @@ jobs:
   rubocop:
     name: Rubocop
     runs-on: ubuntu-latest
+    env:
+      BUNDLE_ONLY: linters
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -21,7 +23,7 @@ jobs:
           bundler-cache: true
 
       - name: rubocop
-        uses: reviewdog/action-rubocop@2f726ae5dd8df72b4faa9d93669cdab96aeb2153
+        uses: reviewdog/action-rubocop@2c8048e3169487eccc1eed812daaa6e5275a809f
         with:
           use_bundler: true
           reporter: github-pr-check
@@ -30,6 +32,8 @@ jobs:
   erb-lint:
     name: ERB Lint runner
     runs-on: ubuntu-latest
+    env:
+      BUNDLE_ONLY: linters
     steps:
       - name: Check out code
         uses: actions/checkout@v4

.rubocop.yml

@@ -6,6 +6,7 @@
 require:
   - rubocop-rails
   - rubocop-rspec
+  - ./rubocop/rubocop
 
 inherit_mode:
   merge:
@@ -18,6 +19,10 @@ AllCops:
 Bundler/OrderedGems:
   Enabled: false
 
+I18n/DeprecatedTranslationKey:
+    Rules:
+      name_with_colon: "Prefer `name` with `mailer.general.metadata_label_indicator` over `name_with_colon`"
+
 Layout/DotPosition:
   EnforcedStyle: leading
 
@@ -72,6 +77,51 @@ Metrics/ParameterLists:
 Metrics/PerceivedComplexity:
   Enabled: false
 
+Migration/LargeTableSchemaUpdate:
+  Tables:
+    - abuse_reports
+    - admin_activities
+    - audits
+    - bookmarks
+    - comments
+    - common_taggings
+    - collection_items
+    - collection_participants
+    - collection_profiles
+    - collections
+    - creatorships
+    - external_works
+    - favorite_tags
+    - feedbacks
+    - filter_counts
+    - filter_taggings
+    - gifts
+    - inbox_comments
+    - invitations
+    - kudos
+    - log_items
+    - mutes
+    - preferences
+    - profiles
+    - prompts
+    - pseuds
+    - readings
+    - set_taggings
+    - serial_works
+    - series
+    - skins
+    - stat_counters
+    - subscriptions
+    - tag_nominations
+    - tag_set_associations
+    - tags
+    - taggings
+    - users
+    - works
+
+Rails/DefaultScope:
+  Enabled: true
+
 Rails/DynamicFindBy:
   AllowedMethods:
     # Exception for Tag.find_by_name

Gemfile

@@ -67,6 +67,10 @@ gem 'devise'
 gem 'devise-async'       # To mails through queues
 gem 'bcrypt'
 
+# Needed for modern ssh
+gem "ed25519", ">= 1.2", "< 2.0"
+gem "bcrypt_pbkdf", ">= 1.0", "< 2.0"
+
 # A highly updated version of the authorization plugin
 gem 'permit_yo'
 gem "pundit"
@@ -111,8 +115,7 @@ gem 'connection_pool'
 gem 'dalli'
 gem 'kgio', '2.10.0'
 
-# TODO: AO3-6297 Update the download code so we can remove mimemagic.
-gem "mimemagic", "0.3.10"
+gem "marcel", "1.0.2"
 
 # Library for helping run pt-online-schema-change commands:
 gem "departure", "~> 6.5"

Gemfile.lock

@@ -122,6 +122,7 @@ GEM
     backports (3.23.0)
     base64 (0.2.0)
     bcrypt (3.1.16)
+    bcrypt_pbkdf (1.1.0)
     better_html (2.0.1)
       actionview (>= 6.0)
       activesupport (>= 6.0)
@@ -222,6 +223,7 @@ GEM
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
+    ed25519 (1.3.0)
     elasticsearch (7.17.1)
       elasticsearch-api (= 7.17.1)
       elasticsearch-transport (= 7.17.1)
@@ -349,10 +351,7 @@ GEM
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
-    mimemagic (0.3.10)
-      nokogiri (~> 1)
-      rake
-    mini_mime (1.1.5)
+    mini_mime (1.1.2)
     mini_portile2 (2.8.5)
     minitest (5.17.0)
     mono_logger (1.1.2)
@@ -378,15 +377,15 @@ GEM
       net-ssh (>= 2.6.5, < 7.0.0)
     net-sftp (3.0.0)
       net-ssh (>= 5.0.0, < 7.0.0)
-    net-smtp (0.4.0)
+    net-smtp (0.5.0)
       net-protocol
     net-ssh (6.1.0)
     net-ssh-gateway (2.0.0)
       net-ssh (>= 4.0.0)
     netrc (0.11.0)
     newrelic_rpm (9.7.1)
     nio4r (2.5.8)
-    nokogiri (1.16.2)
+    nokogiri (1.16.3)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     orm_adapter (0.5.0)
@@ -632,6 +631,7 @@ DEPENDENCIES
   awesome_print
   aws-sdk-s3
   bcrypt
+  bcrypt_pbkdf (>= 1.0, < 2.0)
   brakeman
   bullet (>= 5.7.3)
   bundler
@@ -649,6 +649,7 @@ DEPENDENCIES
   departure (~> 6.5)
   devise
   devise-async
+  ed25519 (>= 1.2, < 2.0)
   elasticsearch (= 7.17.1)
   email_spec (= 1.6.0)
   erb_lint (= 0.4.0)
@@ -669,8 +670,8 @@ DEPENDENCIES
   listen (~> 3.3)
   lograge
   mail (>= 2.8)
+  marcel (= 1.0.2)
   mechanize
-  mimemagic (= 0.3.10)
   minitest
   mysql2
   n_plus_one_control

app/controllers/admin/spam_controller.rb

@@ -1,5 +1,4 @@
 class Admin::SpamController < Admin::BaseController
-
   def index
     authorize ModeratedWork
 
@@ -11,7 +10,11 @@ def index
                   else
                     { reviewed: false, approved: false }
                   end
-    @works = ModeratedWork.where(conditions).order(:created_at).page(params[:page])
+    @works = ModeratedWork.where(conditions)
+      .joins(:work)
+      .includes(:work)
+      .order(:created_at)
+      .page(params[:page])
   end
 
   def bulk_update

app/controllers/autocomplete_controller.rb

@@ -88,18 +88,40 @@ def associated_tags
 
   ## NONCANONICAL TAGS
   def noncanonical_tag
-    search_param = params[:term]
+    search_param = Query.new.escape_reserved_characters(params[:term])
     raise "Redshirt: Attempted to constantize invalid class initialize noncanonical_tag #{params[:type].classify}" unless Tag::TYPES.include?(params[:type].classify)
+
     tag_class = params[:type].classify.constantize
-    render_output(tag_class.by_popularity
-                      .where(["canonical = 0 AND name LIKE ?",
-                              '%' + search_param + '%']).limit(10).map(&:name))
+    one_tag = tag_class.find_by(canonical: false, name: params[:term])
+    # If there is an exact match in the database, ensure it is the first thing suggested.
+    match = if one_tag
+              [one_tag.name]
+            else
+              []
+            end
+
+    # As explained in https://stackoverflow.com/a/54080114, the Elasticsearch suggestion suggester does not support
+    # matches in the middle of a series of words. Therefore, we break the autocomplete query into its individual
+    # words – based on whitespace – except for the last word, which could be incomplete, so a prefix match is
+    # appropriate. This emulates the behavior of SQL `LIKE '%text%'.
+
+    word_list = search_param.split
+    last_word = word_list.pop
+    search_list = word_list.map { |w| { term: { name: { value: w, case_insensitive: true } } } } + [{ prefix: { name: { value: last_word, case_insensitive: true } } }]
+    begin
+      # Size is chosen so we get enough search results from each shard.
+      search_results = $elasticsearch.search(
+        index: TagIndexer.index_name,
+        body: { size: "100", query: { bool: { filter: [{ match: { tag_type: params[:type].capitalize } }, { match: { canonical: false } }], must: search_list } } }
+      )
+      render_output((match + search_results["hits"]["hits"].first(10).map { |t| t["_source"]["name"] }).uniq)
+    rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+      render_output(match)
+    end
   end
 
-
   # more-specific autocompletes should be added below here when they can't be avoided
 
-
   # look up collections ranked by number of items they contain
 
   def collection_fullname

app/controllers/bookmarks_controller.rb

@@ -61,7 +61,7 @@ def search
   def index
     if @bookmarkable
       access_denied unless is_admin? || @bookmarkable.visible?
-      @bookmarks = @bookmarkable.bookmarks.is_public.paginate(page: params[:page], per_page: ArchiveConfig.ITEMS_PER_PAGE)
+      @bookmarks = @bookmarkable.bookmarks.is_public.order_by_created_at.paginate(page: params[:page], per_page: ArchiveConfig.ITEMS_PER_PAGE)
     else
       base_options = {
         show_private: (@user.present? && @user == current_user),
@@ -286,7 +286,7 @@ def fetch_recent
     @bookmarkable = @bookmark.bookmarkable
     respond_to do |format|
       format.js {
-        @bookmarks = @bookmarkable.bookmarks.visible.order("created_at DESC").offset(1).limit(4)
+        @bookmarks = @bookmarkable.bookmarks.order_by_created_at.visible.offset(1).limit(4)
         set_own_bookmarks
       }
       format.html do

app/controllers/home_controller.rb

@@ -1,7 +1,8 @@
 class HomeController < ApplicationController
 
+  before_action :users_only, only: [:first_login_help]
   skip_before_action :store_location, only: [:first_login_help, :token_dispenser]
-  
+
   # unicorn_test
   def unicorn_test
   end
@@ -10,14 +11,14 @@ def unicorn_test
   def tos
     render action: "tos", layout: "application"
   end
-  
+
   # terms of service faq
-  def tos_faq 
+  def tos_faq
     render action: "tos_faq", layout: "application"
   end
 
   # dmca policy
-  def dmca 
+  def dmca
     render action: "dmca", layout: "application"
   end
 
@@ -32,28 +33,28 @@ def token_dispenser
       format.json { render json: { token: form_authenticity_token } }
     end
   end
-  
+
   # diversity statement
-  def diversity 
+  def diversity
     render action: "diversity_statement", layout: "application"
   end
-  
+
   # site map
-  def site_map 
+  def site_map
     render action: "site_map", layout: "application"
   end
-  
+
   # donate
   def donate
     @page_subtitle = t(".page_title")
     render action: "donate", layout: "application"
   end
-  
+
   # about
   def about
     render action: "about", layout: "application"
   end
-  
+
   def first_login_help
     render action: "first_login_help", layout: false
   end

app/controllers/related_works_controller.rb

@@ -65,7 +65,7 @@ def destroy
       end
     end
     @related_work.destroy
-    redirect_back_or_default(user_related_works_path(current_user))
+    redirect_back(fallback_location: user_related_works_path(current_user))
   end
 
   private

app/controllers/series_controller.rb

@@ -22,28 +22,27 @@ def index
     end
     @user = User.find_by!(login: params[:user_id])
     @page_subtitle = ts("%{username} - Series", username: @user.login)
-    pseuds = @user.pseuds
+
+    @series = if current_user.nil?
+                Series.visible_to_all
+              else
+                Series.visible_to_registered_user
+              end
+
     if params[:pseud_id]
-      @pseud = @user.pseuds.find_by!(name: params[:pseud_id])   
+      @pseud = @user.pseuds.find_by!(name: params[:pseud_id])
       @page_subtitle = ts("by ") + @pseud.byline
-      pseuds = [@pseud]
-    end
-
-    if current_user.nil?
-      @series = Series.visible_to_all
+      @series = @series.exclude_anonymous.for_pseud(@pseud)
     else
-      @series = Series.visible_to_registered_user
-    end
-    if pseuds.present?
-      @series = @series.exclude_anonymous.for_pseuds(pseuds)
+      @series = @series.exclude_anonymous.for_user(@user)
     end
     @series = @series.paginate(page: params[:page])
   end
 
   # GET /series/1
   # GET /series/1.xml
   def show
-    @works = @series.works_in_order.posted.select(&:visible?)
+    @works = @series.works_in_order.posted.select(&:visible?).paginate(page: params[:page])
 
     # sets the page title with the data for the series
     @page_title = @series.unrevealed? ? ts("Mystery Series") : get_page_title(@series.allfandoms.collect(&:name).join(', '), @series.anonymous? ? ts("Anonymous") : @series.allpseuds.collect(&:byline).join(', '), @series.title)