From 26d2a2a0cc3d1ea0dfea6810783cd0d8fb0bba10 Mon Sep 17 00:00:00 2001 From: Daniel M Brasil Date: Mon, 23 Oct 2023 12:46:21 -0300 Subject: [PATCH 1/2] Migrate to request specs in `/api/v1/media` (#25543) --- .rubocop_todo.yml | 2 - .../api/v1/media_controller_spec.rb | 107 ---------- spec/requests/api/v1/media_spec.rb | 189 ++++++++++++++++++ 3 files changed, 189 insertions(+), 109 deletions(-) delete mode 100644 spec/controllers/api/v1/media_controller_spec.rb create mode 100644 spec/requests/api/v1/media_spec.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 1a21fd15606f19..9e04be03c26cfa 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -78,7 +78,6 @@ RSpec/AnyInstance: - 'spec/controllers/admin/accounts_controller_spec.rb' - 'spec/controllers/admin/resets_controller_spec.rb' - 'spec/controllers/admin/settings/branding_controller_spec.rb' - - 'spec/controllers/api/v1/media_controller_spec.rb' - 'spec/controllers/auth/sessions_controller_spec.rb' - 'spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb' - 'spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb' @@ -178,7 +177,6 @@ RSpec/LetSetup: RSpec/MessageChain: Exclude: - - 'spec/controllers/api/v1/media_controller_spec.rb' - 'spec/models/concerns/remotable_spec.rb' - 'spec/models/session_activation_spec.rb' - 'spec/models/setting_spec.rb' diff --git a/spec/controllers/api/v1/media_controller_spec.rb b/spec/controllers/api/v1/media_controller_spec.rb deleted file mode 100644 index b574381f90c986..00000000000000 --- a/spec/controllers/api/v1/media_controller_spec.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Api::V1::MediaController do - render_views - - let(:user) { Fabricate(:user) } - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:media') } - - before do - allow(controller).to receive(:doorkeeper_token) { token } - end - - describe 'POST #create' do - describe 'with paperclip errors' do - context 'when imagemagick cant identify the file type' do - it 'returns http 422' do - allow_any_instance_of(Account).to receive_message_chain(:media_attachments, :create!).and_raise(Paperclip::Errors::NotIdentifiedByImageMagickError) - post :create, params: { file: fixture_file_upload('attachment.jpg', 'image/jpeg') } - - expect(response).to have_http_status(422) - end - end - - context 'when there is a generic error' do - it 'returns http 422' do - allow_any_instance_of(Account).to receive_message_chain(:media_attachments, :create!).and_raise(Paperclip::Error) - post :create, params: { file: fixture_file_upload('attachment.jpg', 'image/jpeg') } - - expect(response).to have_http_status(500) - end - end - end - - context 'with image/jpeg' do - before do - post :create, params: { file: fixture_file_upload('attachment.jpg', 'image/jpeg') } - end - - it 'creates a media attachment', :aggregate_failures do - expect(response).to have_http_status(200) - expect(MediaAttachment.first).to_not be_nil - expect(MediaAttachment.first).to have_attached_file(:file) - expect(body_as_json[:id]).to eq MediaAttachment.first.id.to_s - end - end - - context 'with image/gif' do - before do - post :create, params: { file: fixture_file_upload('attachment.gif', 'image/gif') } - end - - it 'creates a media attachment', :aggregate_failures do - expect(response).to have_http_status(200) - expect(MediaAttachment.first).to_not be_nil - expect(MediaAttachment.first).to have_attached_file(:file) - expect(body_as_json[:id]).to eq MediaAttachment.first.id.to_s - end - end - - context 'with video/webm' do - before do - post :create, params: { file: fixture_file_upload('attachment.webm', 'video/webm') } - end - - it 'creates a media attachment', :aggregate_failures do - expect(response).to have_http_status(200) - expect(MediaAttachment.first).to_not be_nil - expect(MediaAttachment.first).to have_attached_file(:file) - expect(body_as_json[:id]).to eq MediaAttachment.first.id.to_s - end - end - end - - describe 'PUT #update' do - context 'when somebody else\'s' do - let(:media) { Fabricate(:media_attachment, status: nil) } - - it 'returns http not found' do - put :update, params: { id: media.id, description: 'Lorem ipsum!!!' } - expect(response).to have_http_status(404) - end - end - - context 'when the author \'s' do - let(:status) { nil } - let(:media) { Fabricate(:media_attachment, status: status, account: user.account) } - - before do - put :update, params: { id: media.id, description: 'Lorem ipsum!!!' } - end - - it 'updates the description' do - expect(media.reload.description).to eq 'Lorem ipsum!!!' - end - - context 'when already attached to a status' do - let(:status) { Fabricate(:status, account: user.account) } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - end - end -end diff --git a/spec/requests/api/v1/media_spec.rb b/spec/requests/api/v1/media_spec.rb new file mode 100644 index 00000000000000..7253a9f1e8560d --- /dev/null +++ b/spec/requests/api/v1/media_spec.rb @@ -0,0 +1,189 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe 'Media' do + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } + let(:scopes) { 'write:media' } + let(:headers) { { 'Authorization' => "Bearer #{token.token}" } } + + describe 'GET /api/v1/media/:id' do + subject do + get "/api/v1/media/#{media.id}", headers: headers + end + + let(:media) { Fabricate(:media_attachment, account: user.account) } + + it_behaves_like 'forbidden for wrong scope', 'read' + + it 'returns http success' do + subject + + expect(response).to have_http_status(200) + end + + it 'returns the media information' do + subject + + expect(body_as_json).to match( + a_hash_including( + id: media.id.to_s, + description: media.description, + type: media.type + ) + ) + end + + context 'when the media is still being processed' do + before do + media.update(processing: :in_progress) + end + + it 'returns http partial content' do + subject + + expect(response).to have_http_status(206) + end + end + + context 'when the media belongs to somebody else' do + let(:media) { Fabricate(:media_attachment) } + + it 'returns http not found' do + subject + + expect(response).to have_http_status(404) + end + end + + context 'when media is attached to a status' do + let(:media) { Fabricate(:media_attachment, account: user.account, status: Fabricate.build(:status)) } + + it 'returns http not found' do + subject + + expect(response).to have_http_status(404) + end + end + end + + describe 'POST /api/v1/media' do + subject do + post '/api/v1/media', headers: headers, params: params + end + + let(:params) { {} } + + shared_examples 'a successful media upload' do |media_type| + it 'uploads the file successfully', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(MediaAttachment.first).to be_present + expect(MediaAttachment.first).to have_attached_file(:file) + end + + it 'returns the correct media content' do + subject + + body = body_as_json + + expect(body).to match( + a_hash_including(id: MediaAttachment.first.id.to_s, description: params[:description], type: media_type) + ) + end + end + + it_behaves_like 'forbidden for wrong scope', 'read read:media' + + describe 'when paperclip errors occur' do + let(:media_attachments) { double } + let(:params) { { file: fixture_file_upload('attachment.jpg', 'image/jpeg') } } + + before do + allow(User).to receive(:find).with(token.resource_owner_id).and_return(user) + allow(user.account).to receive(:media_attachments).and_return(media_attachments) + end + + context 'when imagemagick cannot identify the file type' do + it 'returns http unprocessable entity' do + allow(media_attachments).to receive(:create!).and_raise(Paperclip::Errors::NotIdentifiedByImageMagickError) + + subject + + expect(response).to have_http_status(422) + end + end + + context 'when there is a generic error' do + it 'returns http 500' do + allow(media_attachments).to receive(:create!).and_raise(Paperclip::Error) + + subject + + expect(response).to have_http_status(500) + end + end + end + + context 'with image/jpeg', paperclip_processing: true do + let(:params) { { file: fixture_file_upload('attachment.jpg', 'image/jpeg'), description: 'jpeg image' } } + + it_behaves_like 'a successful media upload', 'image' + end + + context 'with image/gif', paperclip_processing: true do + let(:params) { { file: fixture_file_upload('attachment.gif', 'image/gif') } } + + it_behaves_like 'a successful media upload', 'image' + end + + context 'with video/webm', paperclip_processing: true do + let(:params) { { file: fixture_file_upload('attachment.webm', 'video/webm') } } + + it_behaves_like 'a successful media upload', 'gifv' + end + end + + describe 'PUT /api/v1/media/:id' do + subject do + put "/api/v1/media/#{media.id}", headers: headers, params: params + end + + let(:params) { {} } + let(:media) { Fabricate(:media_attachment, status: status, account: user.account, description: 'old') } + + it_behaves_like 'forbidden for wrong scope', 'read read:media' + + context 'when the media belongs to somebody else' do + let(:media) { Fabricate(:media_attachment, status: nil) } + let(:params) { { description: 'Lorem ipsum!!!' } } + + it 'returns http not found' do + subject + + expect(response).to have_http_status(404) + end + end + + context 'when the requesting user owns the media' do + let(:status) { nil } + let(:params) { { description: 'Lorem ipsum!!!' } } + + it 'updates the description' do + expect { subject }.to change { media.reload.description }.from('old').to('Lorem ipsum!!!') + end + + context 'when the media is attached to a status' do + let(:status) { Fabricate(:status, account: user.account) } + + it 'returns http not found' do + subject + + expect(response).to have_http_status(404) + end + end + end + end +end From 379115e601361c2b5da775fbf28b7dff9dc02e71 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 23 Oct 2023 17:46:21 +0200 Subject: [PATCH 2/2] Add SELF_DESTRUCT env variable to process self-destructions in the background (#26439) --- app/controllers/application_controller.rb | 12 ++++ app/controllers/auth/challenges_controller.rb | 1 + .../auth/confirmations_controller.rb | 1 + .../auth/omniauth_callbacks_controller.rb | 1 + app/controllers/auth/passwords_controller.rb | 1 + .../auth/registrations_controller.rb | 1 + app/controllers/auth/sessions_controller.rb | 1 + app/controllers/backups_controller.rb | 1 + .../concerns/export_controller_concern.rb | 1 + .../settings/exports_controller.rb | 1 + .../settings/login_activities_controller.rb | 3 + .../webauthn_credentials_controller.rb | 1 + ...actor_authentication_methods_controller.rb | 1 + app/helpers/self_destruct_helper.rb | 14 ++++ app/views/auth/registrations/edit.html.haml | 8 ++- app/views/errors/self_destruct.html.haml | 20 ++++++ .../scheduler/self_destruct_scheduler.rb | 72 +++++++++++++++++++ config/initializers/sidekiq.rb | 12 ++++ config/locales/en.yml | 4 ++ config/navigation.rb | 26 +++---- config/sidekiq.yml | 1 + lib/mastodon/cli/main.rb | 65 ++++++----------- 22 files changed, 192 insertions(+), 56 deletions(-) create mode 100644 app/helpers/self_destruct_helper.rb create mode 100644 app/views/errors/self_destruct.html.haml create mode 100644 app/workers/scheduler/self_destruct_scheduler.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 6ec93f824eb570..5f8725f6fc752d 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -12,6 +12,7 @@ class ApplicationController < ActionController::Base include DomainControlHelper include DatabaseHelper include AuthorizedFetchHelper + include SelfDestructHelper helper_method :current_account helper_method :current_session @@ -39,6 +40,8 @@ class ApplicationController < ActionController::Base service_unavailable end + before_action :check_self_destruct! + before_action :store_referrer, except: :raise_not_found, if: :devise_controller? before_action :require_functional!, if: :user_signed_in? @@ -170,6 +173,15 @@ def respond_with_error(code) end end + def check_self_destruct! + return unless self_destruct? + + respond_to do |format| + format.any { render 'errors/self_destruct', layout: 'auth', status: 410, formats: [:html] } + format.json { render json: { error: Rack::Utils::HTTP_STATUS_CODES[410] }, status: code } + end + end + def set_cache_control_defaults response.cache_control.replace(private: true, no_store: true) end diff --git a/app/controllers/auth/challenges_controller.rb b/app/controllers/auth/challenges_controller.rb index 060944240a221a..7ede420b512764 100644 --- a/app/controllers/auth/challenges_controller.rb +++ b/app/controllers/auth/challenges_controller.rb @@ -7,6 +7,7 @@ class Auth::ChallengesController < ApplicationController before_action :authenticate_user! + skip_before_action :check_self_destruct! skip_before_action :require_functional! def create diff --git a/app/controllers/auth/confirmations_controller.rb b/app/controllers/auth/confirmations_controller.rb index 632b624a37e8f0..667e8eb063513e 100644 --- a/app/controllers/auth/confirmations_controller.rb +++ b/app/controllers/auth/confirmations_controller.rb @@ -12,6 +12,7 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController before_action :extend_csp_for_captcha!, only: [:show, :confirm_captcha] before_action :require_captcha_if_needed!, only: [:show] + skip_before_action :check_self_destruct! skip_before_action :require_functional! def show diff --git a/app/controllers/auth/omniauth_callbacks_controller.rb b/app/controllers/auth/omniauth_callbacks_controller.rb index 4723806b9236c1..707b50ef9e1c03 100644 --- a/app/controllers/auth/omniauth_callbacks_controller.rb +++ b/app/controllers/auth/omniauth_callbacks_controller.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController + skip_before_action :check_self_destruct! skip_before_action :verify_authenticity_token def self.provides_callback_for(provider) diff --git a/app/controllers/auth/passwords_controller.rb b/app/controllers/auth/passwords_controller.rb index a8ad669297c8b5..a752194d5b52e2 100644 --- a/app/controllers/auth/passwords_controller.rb +++ b/app/controllers/auth/passwords_controller.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Auth::PasswordsController < Devise::PasswordsController + skip_before_action :check_self_destruct! before_action :check_validity_of_reset_password_token, only: :edit before_action :set_body_classes diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb index b0f2a02aa336f7..331484f36dcc2f 100644 --- a/app/controllers/auth/registrations_controller.rb +++ b/app/controllers/auth/registrations_controller.rb @@ -17,6 +17,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController before_action :require_rules_acceptance!, only: :new before_action :set_registration_form_time, only: :new + skip_before_action :check_self_destruct!, only: [:edit, :update] skip_before_action :require_functional!, only: [:edit, :update] def new diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index 5327192b81d48b..84d9d5e11ed6dd 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -3,6 +3,7 @@ class Auth::SessionsController < Devise::SessionsController layout 'auth' + skip_before_action :check_self_destruct! skip_before_action :require_no_authentication, only: [:create] skip_before_action :require_functional! skip_before_action :update_user_sign_in diff --git a/app/controllers/backups_controller.rb b/app/controllers/backups_controller.rb index db23fefbbcc760..5df1af5f2f7f7c 100644 --- a/app/controllers/backups_controller.rb +++ b/app/controllers/backups_controller.rb @@ -3,6 +3,7 @@ class BackupsController < ApplicationController include RoutingHelper + skip_before_action :check_self_destruct! skip_before_action :require_functional! before_action :authenticate_user! diff --git a/app/controllers/concerns/export_controller_concern.rb b/app/controllers/concerns/export_controller_concern.rb index 24cfc7a0124832..e1792fd6bf2f71 100644 --- a/app/controllers/concerns/export_controller_concern.rb +++ b/app/controllers/concerns/export_controller_concern.rb @@ -7,6 +7,7 @@ module ExportControllerConcern before_action :authenticate_user! before_action :load_export + skip_before_action :check_self_destruct! skip_before_action :require_functional! end diff --git a/app/controllers/settings/exports_controller.rb b/app/controllers/settings/exports_controller.rb index 46a340aeb3dfd4..076ed5dadb178f 100644 --- a/app/controllers/settings/exports_controller.rb +++ b/app/controllers/settings/exports_controller.rb @@ -5,6 +5,7 @@ class Settings::ExportsController < Settings::BaseController include Redisable include Lockable + skip_before_action :check_self_destruct! skip_before_action :require_functional! def show diff --git a/app/controllers/settings/login_activities_controller.rb b/app/controllers/settings/login_activities_controller.rb index 57fa6aef0c8608..50e2d70cb9ad1b 100644 --- a/app/controllers/settings/login_activities_controller.rb +++ b/app/controllers/settings/login_activities_controller.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true class Settings::LoginActivitiesController < Settings::BaseController + skip_before_action :check_self_destruct! + skip_before_action :require_functional! + def index @login_activities = LoginActivity.where(user: current_user).order(id: :desc).page(params[:page]) end diff --git a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb index 4007f134709a65..c86ede4f3adf64 100644 --- a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb +++ b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb @@ -3,6 +3,7 @@ module Settings module TwoFactorAuthentication class WebauthnCredentialsController < BaseController + skip_before_action :check_self_destruct! skip_before_action :require_functional! before_action :require_otp_enabled diff --git a/app/controllers/settings/two_factor_authentication_methods_controller.rb b/app/controllers/settings/two_factor_authentication_methods_controller.rb index 205933ea81482e..a6d5c1fe2dd4f5 100644 --- a/app/controllers/settings/two_factor_authentication_methods_controller.rb +++ b/app/controllers/settings/two_factor_authentication_methods_controller.rb @@ -4,6 +4,7 @@ module Settings class TwoFactorAuthenticationMethodsController < BaseController include ChallengableConcern + skip_before_action :check_self_destruct! skip_before_action :require_functional! before_action :require_challenge!, only: :disable diff --git a/app/helpers/self_destruct_helper.rb b/app/helpers/self_destruct_helper.rb new file mode 100644 index 00000000000000..78557c25e522f2 --- /dev/null +++ b/app/helpers/self_destruct_helper.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module SelfDestructHelper + def self.self_destruct? + value = ENV.fetch('SELF_DESTRUCT', nil) + value.present? && Rails.application.message_verifier('self-destruct').verify(value) == ENV['LOCAL_DOMAIN'] + rescue ActiveSupport::MessageVerifier::InvalidSignature + false + end + + def self_destruct? + SelfDestructHelper.self_destruct? + end +end diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml index 908119a21ad502..f06410b37cfe8b 100644 --- a/app/views/auth/registrations/edit.html.haml +++ b/app/views/auth/registrations/edit.html.haml @@ -1,7 +1,11 @@ - content_for :page_title do = t('settings.account_settings') -= render partial: 'status', locals: { user: @user, strikes: @strikes } +- if self_destruct? + .flash-message.warning + = t('auth.status.self_destruct', domain: ENV['LOCAL_DOMAIN']) +- else + = render partial: 'status', locals: { user: @user, strikes: @strikes } %h3= t('auth.security') @@ -32,7 +36,7 @@ = render partial: 'sessions', object: @sessions -- unless current_account.suspended? +- unless current_account.suspended? || self_destruct? %hr.spacer/ %h3= t('auth.migrate_account') diff --git a/app/views/errors/self_destruct.html.haml b/app/views/errors/self_destruct.html.haml new file mode 100644 index 00000000000000..dc58f1c1353e31 --- /dev/null +++ b/app/views/errors/self_destruct.html.haml @@ -0,0 +1,20 @@ +- content_for :page_title do + = t('self_destruct.title') + +.simple_form + %h1.title= t('self_destruct.title') + %p.lead= t('self_destruct.lead_html', domain: ENV['LOCAL_DOMAIN']) + +.form-footer + %ul.no-list + - if user_signed_in? + %li= link_to t('settings.account_settings'), edit_user_registration_path + - else + - if controller_name != 'sessions' + %li= link_to_login t('auth.login') + + - if controller_name != 'passwords' && controller_name != 'registrations' + %li= link_to t('auth.forgot_password'), new_user_password_path + + - if user_signed_in? + %li= link_to t('auth.logout'), destroy_user_session_path, data: { method: :delete } diff --git a/app/workers/scheduler/self_destruct_scheduler.rb b/app/workers/scheduler/self_destruct_scheduler.rb new file mode 100644 index 00000000000000..d0b6ce8a076d30 --- /dev/null +++ b/app/workers/scheduler/self_destruct_scheduler.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +class Scheduler::SelfDestructScheduler + include Sidekiq::Worker + include SelfDestructHelper + + MAX_ENQUEUED = 10_000 + MAX_REDIS_MEM_USAGE = 0.5 + MAX_ACCOUNT_DELETIONS_PER_JOB = 50 + + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i + + def perform + return unless self_destruct? + return if sidekiq_overwhelmed? + + delete_accounts! + end + + private + + def sidekiq_overwhelmed? + redis_mem_info = Sidekiq.redis_info + + Sidekiq::Stats.new.enqueued > MAX_ENQUEUED || redis_mem_info['used_memory'].to_f > redis_mem_info['total_system_memory'].to_f * MAX_REDIS_MEM_USAGE + end + + def delete_accounts! + # We currently do not distinguish between deleted accounts and suspended + # accounts, and we do not want to remove the records in this scheduler, as + # we still rely on it for account delivery and don't want to perform + # needless work when the database can be outright dropped after the + # self-destruct. + # Deleted accounts are suspended accounts that do not have a pending + # deletion request. + + # This targets accounts that have not been deleted nor marked for deletion yet + Account.local.without_suspended.reorder(id: :asc).take(MAX_ACCOUNT_DELETIONS_PER_JOB).each do |account| + delete_account!(account) + end + + return if sidekiq_overwhelmed? + + # This targets accounts that have been marked for deletion but have not been + # deleted yet + Account.local.suspended.joins(:deletion_request).take(MAX_ACCOUNT_DELETIONS_PER_JOB).each do |account| + delete_account!(account) + account.deletion_request&.destroy + end + end + + def inboxes + @inboxes ||= Account.inboxes + end + + def delete_account!(account) + payload = ActiveModelSerializers::SerializableResource.new( + account, + serializer: ActivityPub::DeleteActorSerializer, + adapter: ActivityPub::Adapter + ).as_json + + json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account)) + + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| + [json, account.id, inbox_url] + end + + # Do not call `Account#suspend!` because we don't want to issue a deletion request + account.update!(suspended_at: Time.now.utc, suspension_origin: :local) + end +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 9d2abf0745eca9..319b386645ff81 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -17,6 +17,18 @@ chain.add SidekiqUniqueJobs::Middleware::Client end + config.on(:startup) do + if SelfDestructHelper.self_destruct? + Sidekiq.schedule = { + 'self_destruct_scheduler' => { + 'interval' => ['1m'], + 'class' => 'Scheduler::SelfDestructScheduler', + 'queue' => 'scheduler', + }, + } + end + end + SidekiqUniqueJobs::Server.configure(config) end diff --git a/config/locales/en.yml b/config/locales/en.yml index cbaec01a4e59a5..e8e0f21e1c3db5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1102,6 +1102,7 @@ en: functional: Your account is fully operational. pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved. redirecting_to: Your account is inactive because it is currently redirecting to %{acct}. + self_destruct: As %{domain} is closing down, you will only get limited access to your account. view_strikes: View past strikes against your account too_fast: Form submitted too fast, try again. use_security_key: Use security key @@ -1572,6 +1573,9 @@ en: over_daily_limit: You have exceeded the limit of %{limit} scheduled posts for today over_total_limit: You have exceeded the limit of %{limit} scheduled posts too_soon: The scheduled date must be in the future + self_destruct: + lead_html: Unfortunately, %{domain} is permanently closing down. If you had an account there, you will not be able to continue using it, but you can still request a backup of your data. + title: This server is closing down sessions: activity: Last activity browser: Browser diff --git a/config/navigation.rb b/config/navigation.rb index e86c695a98db77..1e7acf3b9c61d8 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -1,44 +1,46 @@ # frozen_string_literal: true SimpleNavigation::Configuration.run do |navigation| + self_destruct = SelfDestructHelper.self_destruct? + navigation.items do |n| n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_path n.item :software_updates, safe_join([fa_icon('exclamation-circle fw'), t('admin.critical_update_pending')]), admin_software_updates_path, if: -> { ENV['UPDATE_CHECK_URL'] != '' && current_user.can?(:view_devops) && SoftwareUpdate.urgent_pending? }, html: { class: 'warning' } - n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_path, if: -> { current_user.functional? }, highlights_on: %r{/settings/profile|/settings/featured_tags|/settings/verification|/settings/privacy} + n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_path, if: -> { current_user.functional? && !self_destruct }, highlights_on: %r{/settings/profile|/settings/featured_tags|/settings/verification|/settings/privacy} - n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_path, if: -> { current_user.functional? } do |s| + n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_path, if: -> { current_user.functional? && !self_destruct } do |s| s.item :appearance, safe_join([fa_icon('desktop fw'), t('settings.appearance')]), settings_preferences_appearance_path s.item :notifications, safe_join([fa_icon('bell fw'), t('settings.notifications')]), settings_preferences_notifications_path s.item :other, safe_join([fa_icon('cog fw'), t('preferences.other')]), settings_preferences_other_path end - n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_path, if: -> { current_user.functional? } - n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? } - n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional_or_moved? } + n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_path, if: -> { current_user.functional? && !self_destruct } + n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? && !self_destruct } + n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional_or_moved? && !self_destruct } n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_path do |s| s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_path, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases|/settings/login_activities|^/disputes} s.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_methods_path, highlights_on: %r{/settings/two_factor_authentication|/settings/otp_authentication|/settings/security_keys} - s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_path + s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_path, if: -> { !self_destruct } end n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_path do |s| - s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_imports_path, if: -> { current_user.functional? } + s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_imports_path, if: -> { current_user.functional? && !self_destruct } s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_path end - n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: -> { current_user.can?(:invite_users) && current_user.functional? } - n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_path, if: -> { current_user.functional? } + n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: -> { current_user.can?(:invite_users) && current_user.functional? && !self_destruct } + n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_path, if: -> { current_user.functional? && !self_destruct } - n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_statuses_path, if: -> { current_user.can?(:manage_taxonomies) } do |s| + n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_statuses_path, if: -> { current_user.can?(:manage_taxonomies) && !self_destruct } do |s| s.item :statuses, safe_join([fa_icon('comments-o fw'), t('admin.trends.statuses.title')]), admin_trends_statuses_path, highlights_on: %r{/admin/trends/statuses} s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags} s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links} end - n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks) } do |s| + n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks) && !self_destruct } do |s| s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_path, highlights_on: %r{/admin/reports}, if: -> { current_user.can?(:manage_reports) } s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_path(origin: 'local'), highlights_on: %r{/admin/accounts|/admin/pending_accounts|/admin/disputes|/admin/users}, if: -> { current_user.can?(:manage_users) } s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path, if: -> { current_user.can?(:manage_invites) } @@ -49,7 +51,7 @@ s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_path, if: -> { current_user.can?(:view_audit_log) } end - n.item :admin, safe_join([fa_icon('cogs fw'), t('admin.title')]), nil, if: -> { current_user.can?(:view_dashboard, :manage_settings, :manage_rules, :manage_announcements, :manage_custom_emojis, :manage_webhooks, :manage_federation) } do |s| + n.item :admin, safe_join([fa_icon('cogs fw'), t('admin.title')]), nil, if: -> { current_user.can?(:view_dashboard, :manage_settings, :manage_rules, :manage_announcements, :manage_custom_emojis, :manage_webhooks, :manage_federation) && !self_destruct } do |s| s.item :dashboard, safe_join([fa_icon('tachometer fw'), t('admin.dashboard.title')]), admin_dashboard_path, if: -> { current_user.can?(:view_dashboard) } s.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), admin_settings_path, if: -> { current_user.can?(:manage_settings) }, highlights_on: %r{/admin/settings} s.item :rules, safe_join([fa_icon('gavel fw'), t('admin.rules.title')]), admin_rules_path, highlights_on: %r{/admin/rules}, if: -> { current_user.can?(:manage_rules) } diff --git a/config/sidekiq.yml b/config/sidekiq.yml index f1ba5651dd4e3e..3f9cbd9a7a5bef 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -7,6 +7,7 @@ - [mailers, 2] - [pull] - [scheduler] + :scheduler: :listened_queues_only: true :schedule: diff --git a/lib/mastodon/cli/main.rb b/lib/mastodon/cli/main.rb index 1594eadce81e6a..64f1646f491091 100644 --- a/lib/mastodon/cli/main.rb +++ b/lib/mastodon/cli/main.rb @@ -65,7 +65,6 @@ class Main < Base desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities' subcommand 'maintenance', Maintenance - option :dry_run, type: :boolean desc 'self-destruct', 'Erase the server from the federation' long_desc <<~LONG_DESC Erase the server from the federation by broadcasting account delete @@ -92,55 +91,37 @@ def self_destruct prompt = TTY::Prompt.new - exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain - - unless dry_run? - prompt.warn('This operation WILL NOT be reversible. It can also take a long time.') - prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.') - prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.') - - exit(1) if prompt.no?('Are you sure you want to proceed?') - end + if SelfDestructHelper.self_destruct? + prompt.ok('Self-destruct mode is already enabled for this Mastodon server') - inboxes = Account.inboxes - processed = 0 + pending_accounts = Account.local.without_suspended.count + Account.local.suspended.joins(:deletion_request).count + sidekiq_stats = Sidekiq::Stats.new - Setting.registrations_mode = 'none' unless dry_run? + if pending_accounts.positive? + prompt.warn("#{pending_accounts} accounts are still pending deletion.") + elsif sidekiq_stats.enqueued.positive? + prompt.warn('Deletion notices are still being processed') + elsif sidekiq_stats.retry_size.positive? + prompt.warn('At least one delivery attempt for each deletion notice has been made, but some have failed and are scheduled for retry') + else + prompt.ok('Every deletion notice has been sent! You can safely delete all data and decomission your servers!') + end - if inboxes.empty? - Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless dry_run? - prompt.ok('It seems like your server has not federated with anything') - prompt.ok('You can shut it down and delete it any time') - return + exit(0) end - prompt.warn('Do NOT interrupt this process...') - - delete_account = lambda do |account| - payload = ActiveModelSerializers::SerializableResource.new( - account, - serializer: ActivityPub::DeleteActorSerializer, - adapter: ActivityPub::Adapter - ).as_json - - json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account)) - - unless dry_run? - ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| - [json, account.id, inbox_url] - end - - account.suspend!(block_email: false) - end + exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain - processed += 1 - end + prompt.warn('This operation WILL NOT be reversible.') + prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.') + prompt.warn('The deletion process itself may take a long time, and will be handled by Sidekiq, so do not shut it down until it has finished (you will be able to re-run this command to see the state of the self-destruct process).') - Account.local.without_suspended.find_each { |account| delete_account.call(account) } - Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) } + exit(1) if prompt.no?('Are you sure you want to proceed?') - prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}") - prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data') + self_destruct_value = Rails.application.message_verifier('self-destruct').generate(Rails.configuration.x.local_domain) + prompt.ok('To switch Mastodon to self-destruct mode, add the following variable to your evironment (e.g. by adding a line to your `.env.production`) and restart all Mastodon processes:') + prompt.ok(" SELF_DESTRUCT=#{self_destruct_value}") + prompt.ok("\nYou can re-run this command to see the state of the self-destruct process.") rescue TTY::Reader::InputInterrupt exit(1) end