port tootsuite#11138 to monsterfork: Change domain blocks to automati…

…cally support subdomains

* Change domain blocks to automatically support subdomains

If a more authoritative domain is blocked (example.com), then the
same block will be applied to a subdomain (foo.example.com)

* Match subdomains of existing accounts when blocking/unblocking domains

* Improve code style

app/controllers/admin/domain_blocks_controller.rb

@@ -15,7 +15,7 @@ def create
  resource_params[:domain].strip! if resource_params[:domain].present?
  resource_params[:reason].strip! if resource_params[:reason].present?
  @domain_block = DomainBlock.new(resource_params)
- existing_domain_block = resource_params[:domain].present? ? DomainBlock.find_by(domain: resource_params[:domain].strip) : nil
+ existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
 
  if existing_domain_block.present?
  @domain_block = existing_domain_block

app/controllers/admin/instances_controller.rb

@@ -21,7 +21,7 @@ def show
  @blocks_count = Block.where(target_account: Account.where(domain: params[:id])).count
  @available = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
  @media_storage = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
- @domain_block = DomainBlock.find_by(domain: params[:id])
+ @domain_block = DomainBlock.rule_for(params[:id])
  end
 
  private

app/controllers/media_proxy_controller.rb

@@ -40,4 +40,8 @@ def version
  def lock_options
  { redis: Redis.current, key: "media_download:#{params[:id]}" }
  end
+
+ def reject_media?
+ DomainBlock.reject_media?(@media_attachment.account.domain)
+ end
 end

app/lib/activitypub/activity/create.rb

@@ -486,10 +486,7 @@ def supported_blurhash?(blurhash)
 
  def skip_download?(remote_url = nil)
  return @skip_download if defined?(@skip_download)
- domains = Set[@account.domain]
- domains.add(remote_url.scan(/[\w\-]+\.[\w\-]+(?:\.[\w\-]+)*/).first) if remote_url.present?
- blocks = DomainBlock.suspend.or(DomainBlock.where(reject_media: true))
- @skip_download ||= domains.any? { |domain| blocks.where(domain: domain).or(blocks.where('domain LIKE ?', "%.#{domain}")).exists? }
+ @skip_download ||= DomainBlock.reject_media?(@account.domain)
  end
 
  def reply_to_local?

app/lib/activitypub/activity/flag.rb

@@ -24,7 +24,7 @@ def perform
  private
 
  def skip_reports?
- DomainBlock.find_by(domain: @account.domain)&.reject_reports?
+ DomainBlock.reject_reports?(@account.domain)
  end
 
  def object_uris

app/models/account.rb

@@ -122,6 +122,7 @@ class Account < ApplicationRecord
  scope :popular, -> { order('account_stats.followers_count desc') }
  scope :without_hidden, -> { where(hidden: false) }
  scope :without_unlisted, -> { where(unlisted: false) }
+ scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
  delegate :email,
  :unconfirmed_email,

app/models/custom_emoji.rb

@@ -39,6 +39,7 @@ class CustomEmoji < ApplicationRecord
  scope :local, -> { where(domain: nil) }
  scope :remote, -> { where.not(domain: nil) }
  scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
+ scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
  remotable_attachment :image, LIMIT
 

app/models/domain_block.rb

@@ -33,12 +33,38 @@ class DomainBlock < ApplicationRecord
 
  before_save :set_processing
 
- def self.blocked?(domain)
- suspend.where(domain: domain).or(suspend.where('domain LIKE ?', "%.#{domain}")).exists?
- end
+ class << self
+ def suspend?(domain)
+ !!rule_for(domain)&.suspend?
+ end
+
+ def silence?(domain)
+ !!rule_for(domain)&.silence?
+ end
+
+ def reject_media?(domain)
+ !!rule_for(domain)&.reject_media?
+ end
+
+ def reject_reports?(domain)
+ !!rule_for(domain)&.reject_reports?
+ end
+
+ def force_unlisted?(domain)
+ !!rule_for(domain)&.severity == 'force_unlisted'
+ end
+
+ alias blocked? suspend?
+
+ def rule_for(domain)
+ return if domain.blank?
+
+ uri = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+ segments = uri.normalized_host.split('.')
+ variants = segments.map.with_index { |_, i| segments[i..-1].join('.') }
 
- def self.force_unlisted?(domain)
- where(domain: domain, severity: :force_unlisted).exists?
+  where(domain: variants[0..-2]).order(Arel.sql('char_length(domain) desc')).first
+ end
  end
 
  def stricter_than?(other_block)

app/services/activitypub/process_account_service.rb

@@ -256,8 +256,7 @@ def auto_mark_known?
 
  def domain_block
  return @domain_block if defined?(@domain_block)
-
- @domain_block = DomainBlock.find_by(domain: @domain)
+ @domain_block = DomainBlock.rule_for(@domain)
  end
 
  def key_changed?

app/services/block_domain_service.rb

@@ -123,15 +123,15 @@ def blocked_domain
  end
 
  def blocked_domain_accounts
- Account.where(domain: blocked_domain).reorder(nil)
+ Account.by_domain_and_subdomains(blocked_domain)
  end
 
  def media_from_blocked_domain
  MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
  end
 
  def emojis_from_blocked_domains
- CustomEmoji.where(domain: blocked_domain)
+ CustomEmoji.by_domain_and_subdomains(blocked_domain)
  end
 
  def unknown_accounts

app/services/resolve_account_service.rb

@@ -102,14 +102,19 @@ def process_account!
  end
 
  @account
+ rescue Oj::ParseError
+ nil
  end
 
  def webfinger_update_due?
  @account.nil? || @account.inbox_url.blank? || (!@options[:skip_webfinger] && @account.possibly_stale?)
  end
 
  def activitypub_ready?
- !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type)
+ !@webfinger.link('self').nil? &&
+ ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) &&
+ !actor_json.nil? &&
+ actor_json['inbox'].present?
  end
 
  def actor_url

app/services/unblock_domain_service.rb

@@ -25,7 +25,8 @@ def clear_filtered_status_cache
  end
 
  def blocked_accounts
- scope = Account.where(domain: domain_block.domain)
+ scope = Account.by_domain_and_subdomains(domain_block.domain)
+
  if domain_block.silence?
  scope.where(silenced_at: @domain_block.created_at)
  else

spec/models/domain_block_spec.rb

@@ -21,23 +21,40 @@
  end
  end
 
- describe 'blocked?' do
+ describe '.blocked?' do
  it 'returns true if the domain is suspended' do
- Fabricate(:domain_block, domain: 'domain', severity: :suspend)
- expect(DomainBlock.blocked?('domain')).to eq true
+ Fabricate(:domain_block, domain: 'example.com', severity: :suspend)
+ expect(DomainBlock.blocked?('example.com')).to eq true
  end
 
  it 'returns false even if the domain is silenced' do
- Fabricate(:domain_block, domain: 'domain', severity: :silence)
- expect(DomainBlock.blocked?('domain')).to eq false
+ Fabricate(:domain_block, domain: 'example.com', severity: :silence)
+ expect(DomainBlock.blocked?('example.com')).to eq false
  end
 
  it 'returns false if the domain is not suspended nor silenced' do
- expect(DomainBlock.blocked?('domain')).to eq false
+ expect(DomainBlock.blocked?('example.com')).to eq false
  end
  end
 
- describe 'stricter_than?' do
+ describe '.rule_for' do
+ it 'returns rule matching a blocked domain' do
+ block = Fabricate(:domain_block, domain: 'example.com')
+ expect(DomainBlock.rule_for('example.com')).to eq block
+ end
+
+ it 'returns a rule matching a subdomain of a blocked domain' do
+ block = Fabricate(:domain_block, domain: 'example.com')
+ expect(DomainBlock.rule_for('sub.example.com')).to eq block
+ end
+
+ it 'returns a rule matching a blocked subdomain' do
+ block = Fabricate(:domain_block, domain: 'sub.example.com')
+ expect(DomainBlock.rule_for('sub.example.com')).to eq block
+ end
+ end
+
+ describe '#stricter_than?' do
  it 'returns true if the new block has suspend severity while the old has lower severity' do
  suspend = DomainBlock.new(domain: 'domain', severity: :suspend)
  silence = DomainBlock.new(domain: 'domain', severity: :silence)