Merge branch 'develop' into atsuchanpage

app/controllers/auth/omniauth_callbacks_controller.rb

@@ -7,7 +7,7 @@ def self.provides_callback_for(provider)
     provider_id = provider.to_s.chomp '_oauth2'
 
     define_method provider do
-      @user = User.find_for_oauth(request.env['omniauth.auth'], current_user)
+      @user = User.find_for_omniauth(request.env['omniauth.auth'], current_user)
 
       if @user.persisted?
         LoginActivity.create(
@@ -25,6 +25,9 @@ def self.provides_callback_for(provider)
         session["devise.#{provider}_data"] = request.env['omniauth.auth']
         redirect_to new_user_registration_url
       end
+    rescue ActiveRecord::RecordInvalid
+      flash[:alert] = I18n.t('devise.failure.omniauth_user_creation_failure') if is_navigational_format?
+      redirect_to new_user_session_url
     end
   end
 

app/helpers/jsonld_helper.rb

@@ -168,7 +168,19 @@ def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_tempo
     build_request(uri, on_behalf_of).perform do |response|
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
 
-      body_to_json(response.body_with_limit) if response.code == 200
+      body_to_json(response.body_with_limit) if response.code == 200 && valid_activitypub_content_type?(response)
+    end
+  end
+
+  def valid_activitypub_content_type?(response)
+    return true if response.mime_type == 'application/activity+json'
+
+    # When the mime type is `application/ld+json`, we need to check the profile,
+    # but `http.rb` does not parse it for us.
+    return false unless response.mime_type == 'application/ld+json'
+
+    response.headers[HTTP::Headers::CONTENT_TYPE]&.split(';')&.map(&:strip)&.any? do |str|
+      str.start_with?('profile="') && str[9...-1].split.include?('https://www.w3.org/ns/activitystreams')
     end
   end
 

app/lib/activitypub/activity/create.rb

@@ -42,8 +42,12 @@ def message_franking
     )
   end
 
+  def reject_pattern?
+    Setting.reject_pattern.present? && @object['content']&.match?(Setting.reject_pattern)
+  end
+
   def create_status
-    return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity?
+    return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity? || reject_pattern?
 
     lock_or_fail("create:#{object_uri}") do
       return if delete_arrived_first?(object_uri) || poll_vote?

app/lib/application_extension.rb

@@ -4,12 +4,32 @@ module ApplicationExtension
   extend ActiveSupport::Concern
 
   included do
+    include Redisable
+
     validates :name, length: { maximum: 60 }
     validates :website, url: true, length: { maximum: 2_000 }, if: :website?
     validates :redirect_uri, length: { maximum: 2_000 }
+
+    # The relationship used between Applications and AccessTokens is using
+    # dependent: delete_all, which means the ActiveRecord callback in
+    # AccessTokenExtension is not run, so instead we manually announce to
+    # streaming that these tokens are being deleted.
+    before_destroy :push_to_streaming_api, prepend: true
   end
 
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end
+
+  def push_to_streaming_api
+    # TODO: #28793 Combine into a single topic
+    payload = Oj.dump(event: :kill)
+    access_tokens.in_batches do |tokens|
+      redis.pipelined do |pipeline|
+        tokens.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
+    end
+  end
 end

app/models/concerns/omniauthable.rb

@@ -19,17 +19,18 @@ def email_verified?
   end
 
   class_methods do
-    def find_for_oauth(auth, signed_in_resource = nil)
+    def find_for_omniauth(auth, signed_in_resource = nil)
       # EOLE-SSO Patch
       auth.uid = (auth.uid[0][:uid] || auth.uid[0][:user]) if auth.uid.is_a? Hashie::Array
-      identity = Identity.find_for_oauth(auth)
+      identity = Identity.find_for_omniauth(auth)
 
       # If a signed_in_resource is provided it always overrides the existing user
       # to prevent the identity being locked with accidentally created accounts.
       # Note that this may leave zombie accounts (with no associated identity) which
       # can be cleaned up at a later date.
       user   = signed_in_resource || identity.user
-      user ||= create_for_oauth(auth)
+      user ||= reattach_for_auth(auth)
+      user ||= create_for_auth(auth)
 
       if identity.user.nil?
         identity.user = user
@@ -39,21 +40,35 @@ def find_for_oauth(auth, signed_in_resource = nil)
       user
     end
 
-    def create_for_oauth(auth)
-      # Check if the user exists with provided email if the provider gives us a
-      # verified email.  If no verified email was provided or the user already
-      # exists, we assign a temporary email and ask the user to verify it on
-      # the next step via Auth::SetupController.show
+    private
 
-      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
-      assume_verified   = strategy&.security&.assume_email_is_verified
-      email_is_verified = auth.info.verified || auth.info.verified_email || assume_verified
-      email             = auth.info.verified_email || auth.info.email
-      email             = nil unless email_is_verified
+    def reattach_for_auth(auth)
+      # If allowed, check if a user exists with the provided email address,
+      # and return it if they does not have an associated identity with the
+      # current authentication provider.
 
-      user = User.find_by(email: email) if email_is_verified
+      # This can be used to provide a choice of alternative auth providers
+      # or provide smooth gradual transition between multiple auth providers,
+      # but this is discouraged because any insecure provider will put *all*
+      # local users at risk, regardless of which provider they registered with.
 
-      return user unless user.nil?
+      return unless ENV['ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH'] == 'true'
+
+      email, email_is_verified = email_from_auth(auth)
+      return unless email_is_verified
+
+      user = User.find_by(email: email)
+      return if user.nil? || Identity.exists?(provider: auth.provider, user_id: user.id)
+
+      user
+    end
+
+    def create_for_auth(auth)
+      # Create a user for the given auth params. If no email was provided,
+      # we assign a temporary email and ask the user to verify it on
+      # the next step via Auth::SetupController.show
+
+      email, email_is_verified = email_from_auth(auth)
 
       user = User.new(user_params_from_auth(email, auth))
 
@@ -63,7 +78,14 @@ def create_for_oauth(auth)
       user
     end
 
-    private
+    def email_from_auth(auth)
+      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
+      assume_verified   = strategy&.security&.assume_email_is_verified
+      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
+      email             = auth.info.verified_email || auth.info.email
+
+      [email, email_is_verified]
+    end
 
     def user_params_from_auth(email, auth)
       {

app/models/form/admin_settings.rb

@@ -47,6 +47,7 @@ class Form::AdminSettings
     outgoing_spoilers
     require_invite_text
     captcha_enabled
+    reject_pattern
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -95,6 +96,7 @@ class Form::AdminSettings
   validates :bootstrap_timeline_accounts, existing_username: { multiple: true }
   validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }
   validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }
+  validates :reject_pattern, regexp_syntax: true
 
   def initialize(_attributes = {})
     super

app/models/identity.rb

@@ -16,7 +16,7 @@ class Identity < ApplicationRecord
   validates :uid, presence: true, uniqueness: { scope: :provider }
   validates :provider, presence: true
 
-  def self.find_for_oauth(auth)
+  def self.find_for_omniauth(auth)
     find_or_create_by(uid: auth.uid, provider: auth.provider)
   end
 end

app/models/user.rb

@@ -348,6 +348,16 @@ def reset_password!
     Doorkeeper::AccessToken.by_resource_owner(self).in_batches do |batch|
       batch.update_all(revoked_at: Time.now.utc)
       Web::PushSubscription.where(access_token_id: batch).delete_all
+
+      # Revoke each access token for the Streaming API, since `update_all``
+      # doesn't trigger ActiveRecord Callbacks:
+      # TODO: #28793 Combine into a single topic
+      payload = Oj.dump(event: :kill)
+      redis.pipelined do |pipeline|
+        batch.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
     end
 
     # Finally, send a reset password prompt to the user

app/services/fetch_resource_service.rb

@@ -43,7 +43,7 @@ def process_response(response, terminal = false)
     @response_code = response.code
     return nil if response.code != 200
 
-    if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
+    if valid_activitypub_content_type?(response)
       body = response.body_with_limit
       json = body_to_json(body)
 

app/validators/regexp_syntax_validator.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class RegexpSyntaxValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    return if value.blank?
+
+    begin
+      Regexp.compile(value)
+    rescue RegexpError => exception
+      record.errors.add(attribute, I18n.t('applications.invalid_regexp', message: exception.message))
+    end
+  end
+end

app/views/admin/settings/edit.html.haml

@@ -136,5 +136,8 @@
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
+  .fields-group
+    = f.input :reject_pattern, wrapper: :with_block_label, as: :text, label: t('admin.settings.reject_pattern.title'), hint: t('admin.settings.reject_pattern.desc_html'), input_html: { rows: 8 }
+
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit

config/initializers/doorkeeper.rb

@@ -19,9 +19,14 @@
     user unless user&.otp_required_for_login?
   end
 
-  # If you want to restrict access to the web interface for adding oauth authorized applications, you need to declare the block below.
+  # Doorkeeper provides some administrative interfaces for managing OAuth
+  # Applications, allowing creation, edit, and deletion of applications from the
+  # server. At present, these administrative routes are not integrated into
+  # Mastodon, and as such, we've disabled them by always return a 403 forbidden
+  # response for them. This does not affect the ability for users to manage
+  # their own OAuth Applications.
   admin_authenticator do
-    current_user&.admin? || redirect_to(new_user_session_url)
+    head 403
   end
 
   # Authorization Code expiration time (default 10 minutes).

config/locales/devise.en.yml

@@ -12,6 +12,7 @@ en:
       last_attempt: You have one more attempt before your account is locked.
       locked: Your account is locked.
       not_found_in_database: Invalid %{authentication_keys} or password.
+      omniauth_user_creation_failure: Error creating an account for this identity.
       pending: Your account is still under review.
       timeout: Your session expired. Please sign in again to continue.
       unauthenticated: You need to sign in or sign up before continuing.

config/locales/en.yml

@@ -672,6 +672,9 @@ en:
       mascot:
         desc_html: Displayed on multiple pages. At least 293×205px recommended. When not set, falls back to default mascot
         title: Mascot image
+      reject_pattern:
+        desc_html: Set a regular expression pattern to inspect Create Activity content, and refuse Activity if you match
+        title: Reject Pattern
       peers_api_enabled:
         desc_html: Domain names this server has encountered in the fediverse
         title: Publish list of discovered servers in the API
@@ -854,6 +857,7 @@ en:
   applications:
     created: Application successfully created
     destroyed: Application successfully deleted
+    invalid_regexp: "The provided Regexp is invalid: %{message}"
     invalid_url: The provided URL is invalid
     regenerate_token: Regenerate access token
     token_regenerated: Access token successfully regenerated

config/locales/ja.yml

@@ -664,6 +664,9 @@ ja:
       mascot:
         desc_html: 複数のページに表示されます。サイズは293x205px以上推奨です。未設定の場合、標準のマスコットが使用されます
         title: マスコットイメージ
+      reject_pattern:
+        desc_html: Create Activityのcontentを検査する正規表現パターンを設定し、一致する場合はActivityを拒否します
+        title: 拒否パターン
       peers_api_enabled:
         desc_html: 連合内でこのサーバーが遭遇したドメインの名前
         title: 接続しているサーバーのリストを公開する
@@ -852,6 +855,7 @@ ja:
   applications:
     created: アプリが作成されました
     destroyed: アプリが削除されました
+    invalid_regexp: "正規表現が無効です: %{message}"
     invalid_url: URLが無効です
     regenerate_token: アクセストークンの再生成
     token_regenerated: アクセストークンが再生成されました

config/routes.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'sidekiq_unique_jobs/web'
+require 'sidekiq_unique_jobs/web' if ENV['ENABLE_SIDEKIQ_UNIQUE_JOBS_UI'] == true
 require 'sidekiq-scheduler/web'
 
 Rails.application.routes.draw do

config/settings.yml

@@ -90,6 +90,7 @@ defaults: &defaults
   confirm_follow_from_remote: false
   do_not_allow_follow: false
   auto_accept_followed: true
+  reject_pattern: ''
 
 development:
   <<: *defaults

lib/tasks/sidekiq_unique_jobs.rake

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+namespace :sidekiq_unique_jobs do
+  task delete_all_locks: :environment do
+    digests = SidekiqUniqueJobs::Digests.new
+    digests.delete_by_pattern('*', count: digests.count)
+
+    expiring_digests = SidekiqUniqueJobs::ExpiringDigests.new
+    expiring_digests.delete_by_pattern('*', count: expiring_digests.count)
+  end
+end

spec/helpers/jsonld_helper_spec.rb

@@ -56,36 +56,36 @@
   describe '#fetch_resource' do
     context 'when the second argument is false' do
       it 'returns resource even if the retrieved ID and the given URI does not match' do
-        stub_request(:get, 'https://bob.test/').to_return body: '{"id": "https://alice.test/"}'
-        stub_request(:get, 'https://alice.test/').to_return body: '{"id": "https://alice.test/"}'
+        stub_request(:get, 'https://bob.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
+        stub_request(:get, 'https://alice.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
 
         expect(fetch_resource('https://bob.test/', false)).to eq({ 'id' => 'https://alice.test/' })
       end
 
       it 'returns nil if the object identified by the given URI and the object identified by the retrieved ID does not match' do
-        stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://marvin.test/"}'
-        stub_request(:get, 'https://marvin.test/').to_return body: '{"id": "https://alice.test/"}'
+        stub_request(:get, 'https://mallory.test/').to_return(body: '{"id": "https://marvin.test/"}', headers: { 'Content-Type': 'application/activity+json' })
+        stub_request(:get, 'https://marvin.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
 
         expect(fetch_resource('https://mallory.test/', false)).to eq nil
       end
     end
 
     context 'when the second argument is true' do
       it 'returns nil if the retrieved ID and the given URI does not match' do
-        stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://alice.test/"}'
+        stub_request(:get, 'https://mallory.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
         expect(fetch_resource('https://mallory.test/', true)).to eq nil
       end
     end
   end
 
   describe '#fetch_resource_without_id_validation' do
     it 'returns nil if the status code is not 200' do
-      stub_request(:get, 'https://host.test/').to_return status: 400, body: '{}'
+      stub_request(:get, 'https://host.test/').to_return(status: 400, body: '{}', headers: { 'Content-Type': 'application/activity+json' })
       expect(fetch_resource_without_id_validation('https://host.test/')).to eq nil
     end
 
     it 'returns hash' do
-      stub_request(:get, 'https://host.test/').to_return status: 200, body: '{}'
+      stub_request(:get, 'https://host.test/').to_return(status: 200, body: '{}', headers: { 'Content-Type': 'application/activity+json' })
       expect(fetch_resource_without_id_validation('https://host.test/')).to eq({})
     end
   end