merge upstream (bump version to 3.2.1)

.codeclimate.yml

@@ -30,7 +30,7 @@ plugins:
  channel: eslint-7
  rubocop:
  enabled: true
- channel: rubocop-0-88
+ channel: rubocop-0-92
  sass-lint:
  enabled: true
 exclude_patterns:

Gemfile

@@ -134,7 +134,7 @@ group :test do
 end
 
 group :development do
- gem 'active_record_query_trace', '~> 1.7'
+ gem 'active_record_query_trace', '~> 1.8'
  gem 'annotate', '~> 3.1'
  gem 'better_errors', '~> 2.8'
  gem 'binding_of_caller', '~> 0.7'
@@ -164,3 +164,5 @@ gem 'concurrent-ruby', require: false
 gem 'connection_pool', require: false
 
 gem "sidekiq-statistic", "~> 1.4"
+gem 'xorcist', '~> 1.1'
+gem 'pluck_each', '~> 0.1.3'

Gemfile.lock

@@ -44,7 +44,7 @@ GEM
  activemodel (>= 4.1, < 6.1)
  case_transform (>= 0.2)
  jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
- active_record_query_trace (1.7)
+ active_record_query_trace (1.8)
  activejob (5.2.4.4)
  activesupport (= 5.2.4.4)
  globalid (>= 0.3.6)
@@ -407,6 +407,9 @@ GEM
  pghero (2.7.2)
  activerecord (>= 5)
  pkg-config (1.4.4)
+ pluck_each (0.1.3)
+ activerecord (> 3.2.0)
+ activesupport (> 3.0.0)
  posix-spawn (0.3.15)
  premailer (1.14.2)
  addressable
@@ -670,6 +673,7 @@ GEM
  websocket-extensions (>= 0.1.0)
  websocket-extensions (0.1.5)
  wisper (2.0.1)
+ xorcist (1.1.2)
  xpath (3.2.0)
  nokogiri (~> 1.8)
 
@@ -678,7 +682,7 @@ PLATFORMS
 
 DEPENDENCIES
  active_model_serializers (~> 0.10)
- active_record_query_trace (~> 1.7)
+ active_record_query_trace (~> 1.8)
  addressable (~> 2.7)
  annotate (~> 3.1)
  aws-sdk-s3 (~> 1.83)
@@ -758,6 +762,7 @@ DEPENDENCIES
  pg (~> 1.2)
  pghero (~> 2.7)
  pkg-config (~> 1.4)
+ pluck_each (~> 0.1.3)
  posix-spawn
  premailer-rails
  private_address_check (~> 0.5)
@@ -808,6 +813,7 @@ DEPENDENCIES
  webmock (~> 3.9)
  webpacker (~> 5.2)
  webpush
+ xorcist (~> 1.1)
 
 RUBY VERSION
  ruby 2.6.6p146

app/controllers/activitypub/followers_synchronizations_controller.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ActivityPub::FollowersSynchronizationsController < ActivityPub::BaseController
+ include SignatureVerification
+ include AccountOwnedConcern
+
+ before_action :require_signature!
+ before_action :set_items
+ before_action :set_cache_headers
+
+ def show
+ expires_in 0, public: false
+ render json: collection_presenter,
+ serializer: ActivityPub::CollectionSerializer,
+ adapter: ActivityPub::Adapter,
+ content_type: 'application/activity+json'
+ end
+
+ private
+
+ def uri_prefix
+ signed_request_account.uri[/http(s?):\/\/[^\/]+\//]
+ end
+
+ def set_items
+ @items = @account.followers.where(Account.arel_table[:uri].matches(uri_prefix + '%', false, true)).pluck(:uri)
+ end
+
+ def collection_presenter
+ ActivityPub::CollectionPresenter.new(
+ id: account_followers_synchronization_url(@account),
+ type: :ordered,
+ items: @items
+ )
+ end
+end

app/controllers/activitypub/inboxes_controller.rb

@@ -11,6 +11,7 @@ class ActivityPub::InboxesController < ActivityPub::BaseController
 
  def create
  upgrade_account
+ process_collection_synchronization
  process_payload
  head 202
  end
@@ -52,6 +53,19 @@ def upgrade_account
  DeliveryFailureTracker.reset!(signed_request_account.inbox_url)
  end
 
+ def process_collection_synchronization
+ raw_params = request.headers['Collection-Synchronization']
+ return if raw_params.blank? || ENV['DISABLE_FOLLOWERS_SYNCHRONIZATION'] == 'true'
+
+ # Re-using the syntax for signature parameters
+ tree = SignatureParamsParser.new.parse(raw_params)
+ params = SignatureParamsTransformer.new.apply(tree)
+
+ ActivityPub::PrepareFollowersSynchronizationService.new.call(signed_request_account, params)
+ rescue Parslet::ParseFailed
+ Rails.logger.warn 'Error parsing Collection-Synchronization header'
+ end
+
  def process_payload
  ActivityPub::ProcessingWorker.perform_async(signed_request_account.id, body, @account&.id)
  end

app/lib/activitypub/tag_manager.rb

@@ -40,6 +40,10 @@ def uri_for(target)
  end
  end
 
+ def uri_for_username(username)
+ account_url(username: username)
+ end
+
  def generate_uri_for(_target)
  URI.join(root_url, 'payloads', SecureRandom.uuid)
  end

app/lib/sanitize_config.rb

@@ -18,6 +18,7 @@ module Config
  gopher
  xmpp
  magnet
+ gemini
  ).freeze
 
  CLASS_WHITELIST_TRANSFORMER = lambda do |env|

app/lib/settings/scoped_settings.rb

@@ -11,7 +11,6 @@ def initialize(object)
  @object = object
  end
 
- # rubocop:disable Style/MethodMissingSuper
  def method_missing(method, *args)
  method_name = method.to_s
  # set a value for a variable
@@ -24,7 +23,6 @@ def method_missing(method, *args)
  self[method_name]
  end
  end
- # rubocop:enable Style/MethodMissingSuper
 
  def respond_to_missing?(*)
  true

app/models/account.rb

@@ -352,6 +352,12 @@ def preferred_inbox_url
  shared_inbox_url.presence || inbox_url
  end
 
+ def synchronization_uri_prefix
+ return 'local' if local?
+
+ @synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//]
+ end
+
  class Field < ActiveModelSerializers::Model
  attributes :name, :value, :verified_at, :account, :errors
 

app/models/concerns/account_interactions.rb

@@ -243,6 +243,26 @@ def lists_for_local_distribution
  .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago)
  end
 
+ def remote_followers_hash(url_prefix)
+ Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do
+ digest = "\x00" * 32
+ followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri|
+ Xorcist.xor!(digest, Digest::SHA256.digest(uri))
+ end
+ digest.unpack('H*')[0]
+ end
+ end
+
+ def local_followers_hash
+ Rails.cache.fetch("followers_hash:#{id}:local") do
+ digest = "\x00" * 32
+ followers.where(domain: nil).pluck_each(:username) do |username|
+ Xorcist.xor!(digest, Digest::SHA256.digest(ActivityPub::TagManager.instance.uri_for_username(username)))
+ end
+ digest.unpack('H*')[0]
+ end
+ end
+
  private
 
  def remove_potential_friendship(other_account, mutual = false)

app/models/follow.rb

@@ -41,8 +41,10 @@ def revoke_request!
 
  before_validation :set_uri, only: :create
  after_create :increment_cache_counters
+ after_create :invalidate_hash_cache
  after_destroy :remove_endorsements
  after_destroy :decrement_cache_counters
+ after_destroy :invalidate_hash_cache
 
  private
 
@@ -63,4 +65,10 @@ def decrement_cache_counters
  account&.decrement_count!(:following_count)
  target_account&.decrement_count!(:followers_count)
  end
+
+ def invalidate_hash_cache
+ return if account.local? && target_account.local?
+
+ Rails.cache.delete("followers_hash:#{target_account_id}:#{account.synchronization_uri_prefix}")
+ end
 end

app/services/activitypub/prepare_followers_synchronization_service.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class ActivityPub::PrepareFollowersSynchronizationService < BaseService
+ include JsonLdHelper
+
+ def call(account, params)
+ @account = account
+
+ return if params['collectionId'] != @account.followers_url || invalid_origin?(params['url']) || @account.local_followers_hash == params['digest']
+
+ ActivityPub::FollowersSynchronizationWorker.perform_async(@account.id, params['url'])
+ end
+end

app/services/activitypub/process_account_service.rb

@@ -198,7 +198,7 @@ def collection_info(type)
  total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
  has_first_page = collection.is_a?(Hash) && collection['first'].present?
  @collections[type] = [total_items, has_first_page]
- rescue HTTP::Error, OpenSSL::SSL::SSLError
+ rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::LengthValidationError
  @collections[type] = [nil, nil]
  end
 

app/services/activitypub/synchronize_followers_service.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+class ActivityPub::SynchronizeFollowersService < BaseService
+ include JsonLdHelper
+ include Payloadable
+
+ def call(account, partial_collection_url)
+ @account = account
+
+ items = collection_items(partial_collection_url)
+ return if items.nil?
+
+ # There could be unresolved accounts (hence the call to .compact) but this
+ # should never happen in practice, since in almost all cases we keep an
+ # Account record, and should we not do that, we should have sent a Delete.
+ # In any case there is not much we can do if that occurs.
+ @expected_followers = items.map { |uri| ActivityPub::TagManager.instance.uri_to_resource(uri, Account) }.compact
+
+ remove_unexpected_local_followers!
+ handle_unexpected_outgoing_follows!
+ end
+
+ private
+
+ def remove_unexpected_local_followers!
+ @account.followers.local.where.not(id: @expected_followers.map(&:id)).each do |unexpected_follower|
+ UnfollowService.new.call(unexpected_follower, @account)
+ end
+ end
+
+ def handle_unexpected_outgoing_follows!
+ @expected_followers.each do |expected_follower|
+ next if expected_follower.following?(@account)
+
+ if expected_follower.requested?(@account)
+ # For some reason the follow request went through but we missed it
+ expected_follower.follow_requests.find_by(target_account: @account)&.authorize!
+ else
+ # Since we were not aware of the follow from our side, we do not have an
+ # ID for it that we can include in the Undo activity. For this reason,
+ # the Undo may not work with software that relies exclusively on
+ # matching activity IDs and not the actor and target
+ follow = Follow.new(account: expected_follower, target_account: @account)
+ ActivityPub::DeliveryWorker.perform_async(build_undo_follow_json(follow), follow.account_id, follow.target_account.inbox_url)
+ end
+ end
+ end
+
+ def build_undo_follow_json(follow)
+ Oj.dump(serialize_payload(follow, ActivityPub::UndoFollowSerializer))
+ end
+
+ def collection_items(collection_or_uri)
+ collection = fetch_collection(collection_or_uri)
+ return unless collection.is_a?(Hash)
+
+ collection = fetch_collection(collection['first']) if collection['first'].present?
+ return unless collection.is_a?(Hash)
+
+ case collection['type']
+ when 'Collection', 'CollectionPage'
+ collection['items']
+ when 'OrderedCollection', 'OrderedCollectionPage'
+ collection['orderedItems']
+ end
+ end
+
+ def fetch_collection(collection_or_uri)
+ return collection_or_uri if collection_or_uri.is_a?(Hash)
+ return if invalid_origin?(collection_or_uri)
+
+ fetch_resource_without_id_validation(collection_or_uri, nil, true)
+ end
+end

app/services/process_mentions_service.rb

@@ -60,7 +60,7 @@ def create_notification(mention)
  if mentioned_account.local?
  LocalNotificationWorker.perform_async(mentioned_account.id, mention.id, mention.class.name, :mention)
  elsif mentioned_account.activitypub? && !@status.local_only?
- ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url)
+ ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url, { synchronize_followers: !mention.status.distributable? })
  end
  end
 

app/workers/activitypub/delivery_worker.rb

@@ -2,6 +2,7 @@
 
 class ActivityPub::DeliveryWorker
  include Sidekiq::Worker
+ include RoutingHelper
  include JsonLdHelper
 
  STOPLIGHT_FAILURE_THRESHOLD = 10
@@ -38,9 +39,18 @@ def build_request(http_client)
  Request.new(:post, @inbox_url, body: @json, http_client: http_client).tap do |request|
  request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
  request.add_headers(HEADERS)
+ request.add_headers({ 'Collection-Synchronization' => synchronization_header }) if ENV['DISABLE_FOLLOWERS_SYNCHRONIZATION'] != 'true' && @options[:synchronize_followers]
  end
  end
 
+ def synchronization_header
+ "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
+ end
+
+ def inbox_url_prefix
+ @inbox_url[/http(s?):\/\/[^\/]+\//]
+ end
+
  def perform_request
  light = Stoplight(@inbox_url) do
  request_pool.with(@host) do |http_client|

app/workers/activitypub/distribution_worker.rb

@@ -13,7 +13,7 @@ def perform(status_id)
  return if skip_distribution?
 
  ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
- [payload, @account.id, inbox_url]
+ [payload, @account.id, inbox_url, { synchronize_followers: !@status.distributable? }]
  end
 
  relay! if relayable?

app/workers/activitypub/followers_synchronization_worker.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class ActivityPub::FollowersSynchronizationWorker
+ include Sidekiq::Worker
+
+ sidekiq_options queue: 'push', lock: :until_executed
+
+ def perform(account_id, url)
+ @account = Account.find_by(id: account_id)
+ return true if @account.nil?
+
+ ActivityPub::SynchronizeFollowersService.new.call(@account, url)
+ end
+end