Merge branch 'develop' into atsuchanpage

Gemfile

@@ -158,3 +158,5 @@ gem 'connection_pool', require: false
 gem 'xorcist', '~> 1.1'
 
 gem 'hcaptcha', '~> 7.1'
+gem 'cocoon', '~> 1.2'
+gem 'mail', '~> 2.8'

Gemfile.lock

@@ -155,6 +155,7 @@ GEM
     cld3 (3.4.4)
       ffi (>= 1.1.0, < 1.16.0)
     climate_control (0.2.0)
+    cocoon (1.2.15)
     coderay (1.1.3)
     color_diff (0.1)
     concurrent-ruby (1.1.9)
@@ -167,6 +168,7 @@ GEM
     crass (1.0.6)
     css_parser (1.7.1)
       addressable
+    date (3.3.4)
     debug_inspector (1.0.0)
     devise (4.8.1)
       bcrypt (~> 3.0)
@@ -358,8 +360,11 @@ GEM
     loofah (2.13.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    mail (2.7.1)
+    mail (2.8.1)
       mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
     makara (0.5.1)
       activerecord (>= 5.2.0)
     marcel (1.0.1)
@@ -380,9 +385,18 @@ GEM
     msgpack (1.4.4)
     multi_json (1.15.0)
     multipart-post (2.1.1)
+    net-imap (0.4.14)
+      date
+      net-protocol
     net-ldap (0.17.0)
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
     net-scp (3.0.0)
       net-ssh (>= 2.6.5, < 7.0.0)
+    net-smtp (0.5.0)
+      net-protocol
     net-ssh (6.1.0)
     nio4r (2.5.8)
     nokogiri (1.13.1)
@@ -619,6 +633,7 @@ GEM
     thwait (0.2.0)
       e2mmap
     tilt (2.0.10)
+    timeout (0.4.1)
     tpm-key_attestation (0.9.0)
       bindata (~> 2.4)
       openssl-signature_algorithm (~> 0.4.0)
@@ -703,6 +718,7 @@ DEPENDENCIES
   chewy (~> 7.2)
   cld3 (~> 3.4.4)
   climate_control (~> 0.2)
+  cocoon (~> 1.2)
   color_diff (~> 0.1)
   concurrent-ruby
   connection_pool
@@ -738,6 +754,7 @@ DEPENDENCIES
   letter_opener_web (~> 2.0)
   link_header (~> 0.0)
   lograge (~> 0.11)
+  mail (~> 2.8)
   makara (~> 0.5)
   mario-redis-lock (~> 1.2)
   memory_profiler

app/controllers/admin/domain_allows_controller.rb

@@ -25,6 +25,8 @@ def create
   def destroy
     authorize @domain_allow, :destroy?
     UnallowDomainService.new.call(@domain_allow)
+    log_action :destroy, @domain_allow
+
     redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
   end
 

app/controllers/api/v1/featured_tags/suggestions_controller.rb

@@ -12,6 +12,10 @@ def index
   private
 
   def set_recently_used_tags
-    @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10)
+    @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10)
+  end
+
+  def featured_tag_ids
+    current_account.featured_tags.pluck(:tag_id)
   end
 end

app/controllers/oauth/authorized_applications_controller.rb

@@ -15,6 +15,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
 
   def destroy
     Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner)
+    Doorkeeper::Application.find_by(id: params[:id])&.close_streaming_sessions(current_resource_owner)
     super
   end
 

app/controllers/well_known/webfinger_controller.rb

@@ -18,7 +18,14 @@ def show
     private
 
     def set_account
-      @account = Account.find_local!(username_from_resource)
+      username = username_from_resource
+      @account = begin
+        if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain
+          Account.representative
+        else
+          Account.find_local!(username)
+        end
+      end
     end
 
     def username_from_resource

app/lib/activitypub/activity/create.rb

@@ -104,7 +104,7 @@ def distribute
   def find_existing_status
     status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
-    status
+    status if status&.account_id == @account.id
   end
 
   def process_status_params

app/lib/application_extension.rb

@@ -14,17 +14,19 @@ module ApplicationExtension
     # dependent: delete_all, which means the ActiveRecord callback in
     # AccessTokenExtension is not run, so instead we manually announce to
     # streaming that these tokens are being deleted.
-    before_destroy :push_to_streaming_api, prepend: true
+    before_destroy :close_streaming_sessions, prepend: true
   end
 
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end
 
-  def push_to_streaming_api
+  def close_streaming_sessions(resource_owner = nil)
     # TODO: #28793 Combine into a single topic
     payload = Oj.dump(event: :kill)
-    access_tokens.in_batches do |tokens|
+    scope = access_tokens
+    scope = scope.where(resource_owner_id: resource_owner.id) unless resource_owner.nil?
+    scope.in_batches do |tokens|
       redis.pipelined do |pipeline|
         tokens.ids.each do |id|
           pipeline.publish("timeline:access_token:#{id}", payload)

app/lib/search_query_transformer.rb

@@ -70,7 +70,7 @@ def initialize(prefix, operator, phrase)
   end
 
   rule(clause: subtree(:clause)) do
-    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    prefix   = clause[:prefix][:term].to_s.downcase if clause[:prefix]
     operator = clause[:operator]&.to_s
 
     if clause[:term]

app/lib/video_metadata_extractor.rb

@@ -22,7 +22,7 @@ def valid?
   private
 
   def ffmpeg_command_output
-    command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
+    command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
     command.run(path: @path, format: 'json', loglevel: 'fatal')
   end
 

app/models/concerns/account_interactions.rb

@@ -179,7 +179,7 @@ def unmute_conversation!(conversation)
   end
 
   def unblock_domain!(other_domain)
-    block = domain_blocks.find_by(domain: other_domain)
+    block = domain_blocks.find_by(domain: normalized_domain(other_domain))
     block&.destroy
   end
 
@@ -287,4 +287,8 @@ def remove_potential_friendship(other_account, mutual = false)
     PotentialFriendshipTracker.remove(id, other_account.id)
     PotentialFriendshipTracker.remove(other_account.id, id) if mutual
   end
+
+  def normalized_domain(domain)
+    TagManager.instance.normalize_domain(domain)
+  end
 end

app/models/status.rb

@@ -176,7 +176,7 @@ def in_reply_to_local_account?
   end
 
   def reblog?
-    !reblog_of_id.nil?
+    !reblog_of_id.nil? && reblog
   end
 
   def within_realtime_window?

app/models/user.rb

@@ -91,7 +91,10 @@ class User < ApplicationRecord
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
-  validates_with BlacklistedEmailValidator, if: -> { !confirmed? }
+
+  validates :email, presence: true, email_address: true
+
+  validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create
 

app/services/fetch_link_card_service.rb

@@ -12,6 +12,9 @@ class FetchLinkCardService < BaseService
     )
   }iox
 
+  # URL size limit to safely store in PosgreSQL's unique indexes
+  BYTESIZE_LIMIT = 2692
+
   def call(status)
     @status       = status
     @original_url = parse_urls
@@ -86,7 +89,7 @@ def parse_urls
 
   def bad_url?(uri)
     # Avoid local instance URLs and invalid URLs
-    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
+    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT
   end
 
   def mention_link?(anchor)

app/services/notify_service.rb

@@ -76,47 +76,24 @@ def response_to_recipient?
     return false if @notification.target_status.in_reply_to_id.nil?
 
     # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads
-    !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero?
-      WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender, path) AS (
-          SELECT
-            s.id,
-            s.in_reply_to_id,
-            (CASE
-              WHEN s.account_id = :recipient_id THEN
-                EXISTS (
-                  SELECT *
-                  FROM mentions m
-                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
-                )
-              ELSE
-                FALSE
-             END),
-            ARRAY[s.id]
+    !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id, depth_limit: 100]).zero?
+      WITH RECURSIVE ancestors(id, in_reply_to_id, mention_id, path, depth) AS (
+          SELECT s.id, s.in_reply_to_id, m.id, ARRAY[s.id], 0
           FROM statuses s
+          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
           WHERE s.id = :id
         UNION ALL
-          SELECT
-            s.id,
-            s.in_reply_to_id,
-            (CASE
-              WHEN s.account_id = :recipient_id THEN
-                EXISTS (
-                  SELECT *
-                  FROM mentions m
-                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
-                )
-              ELSE
-                FALSE
-             END),
-            st.path || s.id
-          FROM ancestors st
-          JOIN statuses s ON s.id = st.in_reply_to_id
-          WHERE st.replying_to_sender IS FALSE AND NOT s.id = ANY(path)
+          SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1
+          FROM ancestors
+          JOIN statuses s ON s.id = ancestors.in_reply_to_id
+          /* early exit if we already have a mention matching our requirements */
+          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id
+          WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit
       )
       SELECT COUNT(*)
-      FROM ancestors st
-      JOIN statuses s ON s.id = st.id
-      WHERE st.replying_to_sender IS TRUE AND s.visibility = 3
+      FROM ancestors
+      JOIN statuses s ON s.id = ancestors.id
+      WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3
     SQL
   end
 

app/validators/email_address_validator.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# NOTE: I initially wrote this as `EmailValidator` but it ended up clashing
+# with an indirect dependency of ours, `validate_email`, which, turns out,
+# has the same approach as we do, but with an extra check disallowing
+# single-label domains. Decided to not switch to `validate_email` because
+# we do want to allow at least `localhost`.
+
+class EmailAddressValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    value = value.strip
+
+    address = Mail::Address.new(value)
+    record.errors.add(attribute, :invalid) if address.address != value
+  rescue Mail::Field::FieldError
+    record.errors.add(attribute, :invalid)
+  end
+end

app/workers/mute_worker.rb

@@ -5,6 +5,7 @@ class MuteWorker
 
   def perform(account_id, target_account_id)
     FeedManager.instance.clear_from_home(Account.find(account_id), Account.find(target_account_id))
+    FeedManager.instance.clear_from_lists(Account.find(account_id), Account.find(target_account_id))
   rescue ActiveRecord::RecordNotFound
     true
   end

config/application.rb

@@ -44,6 +44,7 @@
 require_relative '../lib/webpacker/helper_extensions'
 require_relative '../lib/action_dispatch/cookie_jar_extensions'
 require_relative '../lib/rails/engine_extensions'
+require_relative '../lib/action_dispatch/remote_ip_extensions'
 require_relative '../lib/active_record/database_tasks_extensions'
 require_relative '../lib/active_record/batches'
 

config/initializers/ffmpeg.rb

@@ -1,3 +1,6 @@
-if ENV['FFMPEG_BINARY'].present?
-    FFMPEG.ffmpeg_binary = ENV['FFMPEG_BINARY']
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.ffmpeg_binary = ENV['FFMPEG_BINARY'] || 'ffmpeg'
+  config.x.ffprobe_binary = ENV['FFPROBE_BINARY'] || 'ffprobe'
 end

config/initializers/rack_attack.rb

@@ -33,6 +33,14 @@ def authenticated_user_id
       authenticated_token&.resource_owner_id
     end
 
+    def authenticated_token_id
+      authenticated_token&.id
+    end
+
+    def warden_user_id
+      @env['warden']&.user&.id
+    end
+
     def unauthenticated?
       !authenticated_user_id
     end
@@ -54,10 +62,6 @@ def paging_request?
     end
   end
 
-  Rack::Attack.safelist('allow from localhost') do |req|
-    req.remote_ip == '127.0.0.1' || req.remote_ip == '::1'
-  end
-
   Rack::Attack.blocklist('deny from blocklist') do |req|
     IpBlock.blocked?(req.remote_ip)
   end
@@ -129,6 +133,10 @@ def paging_request?
     req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in')
   end
 
+  throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req|
+    req.warden_user_id if (req.put? || req.patch?) && req.path_matches?('/auth')
+  end
+
   self.throttled_response = lambda do |env|
     now        = Time.now.utc
     match_data = env['rack.attack.match_data']

config/initializers/statsd.rb

@@ -3,13 +3,17 @@
 if ENV['STATSD_ADDR'].present?
   host, port = ENV['STATSD_ADDR'].split(':')
 
-  $statsd = ::Statsd.new(host, port)
-  $statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }
+  begin
+    statsd = Statsd.new(host, port)
+    statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }
 
-  ::NSA.inform_statsd($statsd) do |informant|
-    informant.collect(:action_controller, :web)
-    informant.collect(:active_record, :db)
-    informant.collect(:active_support_cache, :cache)
-    informant.collect(:sidekiq, :sidekiq)
+    NSA.inform_statsd(statsd) do |informant|
+      informant.collect(:action_controller, :web)
+      informant.collect(:active_record, :db)
+      informant.collect(:active_support_cache, :cache)
+      informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true'
+    end
+  rescue
+    Rails.logger.warn("statsd address #{ENV['STATSD_ADDR']} not reachable, proceeding without statsd")
   end
 end