diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index bddfd2d27a3942..00000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,225 +0,0 @@ -version: 2.1 - -orbs: - ruby: circleci/ruby@1.4.1 - node: circleci/node@5.0.1 - -executors: - default: - parameters: - ruby-version: - type: string - docker: - - image: cimg/ruby:<< parameters.ruby-version >> - environment: - BUNDLE_JOBS: 3 - BUNDLE_RETRY: 3 - CONTINUOUS_INTEGRATION: true - DB_HOST: localhost - DB_USER: root - DISABLE_SIMPLECOV: true - RAILS_ENV: test - - image: cimg/postgres:14.0 - environment: - POSTGRES_USER: root - POSTGRES_HOST_AUTH_METHOD: trust - - image: cimg/redis:6.2 - -commands: - install-system-dependencies: - steps: - - run: - name: Install system dependencies - command: | - sudo apt-get update - sudo apt-get install -y libicu-dev libidn11-dev - install-ruby-dependencies: - parameters: - ruby-version: - type: string - steps: - - run: - command: | - bundle config clean 'true' - bundle config frozen 'true' - bundle config without 'development production' - name: Set bundler settings - - ruby/install-deps: - bundler-version: '2.3.8' - key: ruby<< parameters.ruby-version >>-gems-v1 - wait-db: - steps: - - run: - command: dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m - name: Wait for PostgreSQL and Redis - -jobs: - build: - docker: - - image: cimg/ruby:3.0-node - environment: - RAILS_ENV: test - steps: - - checkout - - install-system-dependencies - - install-ruby-dependencies: - ruby-version: '3.0' - - node/install-packages: - cache-version: v1 - pkg-manager: yarn - - run: - command: | - export NODE_OPTIONS=--openssl-legacy-provider - ./bin/rails assets:precompile - name: Precompile assets - - persist_to_workspace: - paths: - - public/assets - - public/packs-test - root: . - - test: - parameters: - ruby-version: - type: string - executor: - name: default - ruby-version: << parameters.ruby-version >> - environment: - ALLOW_NOPAM: true - PAM_ENABLED: true - PAM_DEFAULT_SERVICE: pam_test - PAM_CONTROLLED_SERVICE: pam_test_controlled - parallelism: 4 - steps: - - checkout - - install-system-dependencies - - run: - command: sudo apt-get install -y ffmpeg imagemagick libpam-dev - name: Install additional system dependencies - - run: - command: bundle config with 'pam_authentication' - name: Enable PAM authentication - - install-ruby-dependencies: - ruby-version: << parameters.ruby-version >> - - attach_workspace: - at: . - - wait-db - - run: - command: ./bin/rails db:create db:schema:load db:seed - name: Load database schema - - ruby/rspec-test - - test-migrations: - executor: - name: default - ruby-version: '3.0' - steps: - - checkout - - install-system-dependencies - - install-ruby-dependencies: - ruby-version: '3.0' - - wait-db - - run: - command: ./bin/rails db:create - name: Create database - - run: - command: ./bin/rails db:migrate VERSION=20171010025614 - name: Run migrations up to v2.0.0 - - run: - command: ./bin/rails tests:migrations:populate_v2 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate VERSION=20180514140000 - name: Run migrations up to v2.4.0 - - run: - command: ./bin/rails tests:migrations:populate_v2_4 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate VERSION=20180707154237 - name: Run migrations up to v2.4.3 - - run: - command: ./bin/rails tests:migrations:populate_v2_4_3 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate - name: Run all remaining migrations - - run: - command: ./bin/rails tests:migrations:check_database - name: Check migration result - - test-two-step-migrations: - executor: - name: default - ruby-version: '3.0' - steps: - - checkout - - install-system-dependencies - - install-ruby-dependencies: - ruby-version: '3.0' - - wait-db - - run: - command: ./bin/rails db:create - name: Create database - - run: - command: ./bin/rails db:migrate VERSION=20171010025614 - name: Run migrations up to v2.0.0 - - run: - command: ./bin/rails tests:migrations:populate_v2 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate VERSION=20180514140000 - name: Run pre-deployment migrations up to v2.4.0 - environment: - SKIP_POST_DEPLOYMENT_MIGRATIONS: true - - run: - command: ./bin/rails tests:migrations:populate_v2_4 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate VERSION=20180707154237 - name: Run migrations up to v2.4.3 - environment: - SKIP_POST_DEPLOYMENT_MIGRATIONS: true - - run: - command: ./bin/rails tests:migrations:populate_v2_4_3 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate - name: Run all remaining pre-deployment migrations - environment: - SKIP_POST_DEPLOYMENT_MIGRATIONS: true - - run: - command: ./bin/rails db:migrate - name: Run all post-deployment migrations - - run: - command: ./bin/rails tests:migrations:check_database - name: Check migration result - -workflows: - version: 2 - build-and-test: - jobs: - - build - - test: - matrix: - parameters: - ruby-version: - - '2.7' - - '3.0' - name: test-ruby<< matrix.ruby-version >> - requires: - - build - - test-migrations: - requires: - - build - - test-two-step-migrations: - requires: - - build - - node/run: - cache-version: v1 - name: test-webui - pkg-manager: yarn - requires: - - build - version: lts - yarn-run: test:jest diff --git a/.github/workflows/build-container-image.yml b/.github/workflows/build-container-image.yml index 5998794601afdd..b9aebcc46c60d3 100644 --- a/.github/workflows/build-container-image.yml +++ b/.github/workflows/build-container-image.yml @@ -4,6 +4,9 @@ on: platforms: required: true type: string + cache: + type: boolean + default: true use_native_arm64_builder: type: boolean push_to_images: @@ -85,5 +88,5 @@ jobs: push: ${{ inputs.push_to_images != '' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: ${{ inputs.cache && 'type=gha' || '' }} + cache-to: ${{ inputs.cache && 'type=gha,mode=max' || '' }} diff --git a/.github/workflows/build-releases.yml b/.github/workflows/build-releases.yml index 72a12f4dffad43..c19766b1862ff6 100644 --- a/.github/workflows/build-releases.yml +++ b/.github/workflows/build-releases.yml @@ -17,6 +17,8 @@ jobs: push_to_images: | tootsuite/mastodon ghcr.io/mastodon/mastodon + # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages + cache: false flavor: | latest=false tags: | diff --git a/.github/workflows/test-image-build.yml b/.github/workflows/test-image-build.yml new file mode 100644 index 00000000000000..71344c0046aa01 --- /dev/null +++ b/.github/workflows/test-image-build.yml @@ -0,0 +1,15 @@ +name: Test container image build +on: + pull_request: +permissions: + contents: read + +jobs: + build-image: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + uses: ./.github/workflows/build-container-image.yml + with: + platforms: linux/amd64 # Testing only on native platform so it is performant diff --git a/CHANGELOG.md b/CHANGELOG.md index 583704e50731dc..36eb51a5145d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,51 @@ All notable changes to this project will be documented in this file. ## End of life notice -**The 4.0.x branch will not receive any update after 2023-10-31.** -This means that no security fix will be made available for this branch after this date, and you will need to update to a more recent version (such as the 4.1.x branch) to receive security fixes. +**The 4.0.x branch has reached its end of life and will not receive any further update.** +This means that no security fix will be made available for this branch after this date, and you will need to update to a more recent version (such as the 4.2.x branch) to receive security fixes. + +## [4.0.13] - 2024-02-01 + +### Security + +- Fix insufficient origin validation (CVE-2024-23832, [GHSA-3fjr-858r-92rw](https://github.com/mastodon/mastodon/security/advisories/GHSA-3fjr-858r-92rw)) + +## [4.0.12] - 2023-10-10 + +### Changed + +- Change some worker lock TTLs to be shorter-lived ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27246)) +- Change user archive export allowed period from 7 days to 6 days ([suddjian](https://github.com/mastodon/mastodon/pull/27200)) + +### Fixed + +- Fix mentions being matched in some URL query strings ([mjankowski](https://github.com/mastodon/mastodon/pull/25656)) +- Fix multiple instances of the trend refresh scheduler sometimes running at once ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27253)) +- Fix importer returning negative row estimates ([jgillich](https://github.com/mastodon/mastodon/pull/27258)) +- Fix filtering audit log for entries about disabling 2FA ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27186)) +- Fix tIME chunk not being properly removed from PNG uploads ([TheEssem](https://github.com/mastodon/mastodon/pull/27111)) +- Fix inefficient queries in “Follows and followers” as well as several admin pages ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27116), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27306)) + +## [4.0.11] - 2023-09-20 + +### Fixed + +- Fix post translation erroring out ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26990)) + +## [4.0.10] - 2023-09-19 + +### Fixed + +- Fix moderator rights inconsistencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26729)) +- Fix crash when encountering invalid URL ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26814)) +- Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409)) +- Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500)) +- Fix unexpected audio stream transcoding when uploaded video is eligible to passthrough ([yufushiro](https://github.com/mastodon/mastodon/pull/26608)) + +### Security + +- Fix missing HTML sanitization in translation API (CVE-2023-42452) +- Fix incorrect domain name normalization (CVE-2023-42451) ## [4.0.9] - 2023-09-05 diff --git a/Dockerfile b/Dockerfile index 172e1cd2c1871e..577cd08450b98d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ RUN ARCH= && \ esac && \ echo "Etc/UTC" > /etc/localtime && \ apt-get update && \ + apt-get -yq dist-upgrade && \ apt-get install -y --no-install-recommends ca-certificates wget python3 apt-utils && \ cd ~ && \ wget -q https://nodejs.org/download/release/v$NODE_VER/node-v$NODE_VER-linux-$ARCH.tar.gz && \ diff --git a/Gemfile.lock b/Gemfile.lock index b47f840ca32870..a5731cd3111b5b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,40 +10,40 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (6.1.7.4) - actionpack (= 6.1.7.4) - activesupport (= 6.1.7.4) + actioncable (6.1.7.6) + actionpack (= 6.1.7.6) + activesupport (= 6.1.7.6) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.7.4) - actionpack (= 6.1.7.4) - activejob (= 6.1.7.4) - activerecord (= 6.1.7.4) - activestorage (= 6.1.7.4) - activesupport (= 6.1.7.4) + actionmailbox (6.1.7.6) + actionpack (= 6.1.7.6) + activejob (= 6.1.7.6) + activerecord (= 6.1.7.6) + activestorage (= 6.1.7.6) + activesupport (= 6.1.7.6) mail (>= 2.7.1) - actionmailer (6.1.7.4) - actionpack (= 6.1.7.4) - actionview (= 6.1.7.4) - activejob (= 6.1.7.4) - activesupport (= 6.1.7.4) + actionmailer (6.1.7.6) + actionpack (= 6.1.7.6) + actionview (= 6.1.7.6) + activejob (= 6.1.7.6) + activesupport (= 6.1.7.6) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.7.4) - actionview (= 6.1.7.4) - activesupport (= 6.1.7.4) + actionpack (6.1.7.6) + actionview (= 6.1.7.6) + activesupport (= 6.1.7.6) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.7.4) - actionpack (= 6.1.7.4) - activerecord (= 6.1.7.4) - activestorage (= 6.1.7.4) - activesupport (= 6.1.7.4) + actiontext (6.1.7.6) + actionpack (= 6.1.7.6) + activerecord (= 6.1.7.6) + activestorage (= 6.1.7.6) + activesupport (= 6.1.7.6) nokogiri (>= 1.8.5) - actionview (6.1.7.4) - activesupport (= 6.1.7.4) + actionview (6.1.7.6) + activesupport (= 6.1.7.6) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -54,22 +54,22 @@ GEM case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) active_record_query_trace (1.8) - activejob (6.1.7.4) - activesupport (= 6.1.7.4) + activejob (6.1.7.6) + activesupport (= 6.1.7.6) globalid (>= 0.3.6) - activemodel (6.1.7.4) - activesupport (= 6.1.7.4) - activerecord (6.1.7.4) - activemodel (= 6.1.7.4) - activesupport (= 6.1.7.4) - activestorage (6.1.7.4) - actionpack (= 6.1.7.4) - activejob (= 6.1.7.4) - activerecord (= 6.1.7.4) - activesupport (= 6.1.7.4) + activemodel (6.1.7.6) + activesupport (= 6.1.7.6) + activerecord (6.1.7.6) + activemodel (= 6.1.7.6) + activesupport (= 6.1.7.6) + activestorage (6.1.7.6) + actionpack (= 6.1.7.6) + activejob (= 6.1.7.6) + activerecord (= 6.1.7.6) + activesupport (= 6.1.7.6) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.7.4) + activesupport (6.1.7.6) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -174,7 +174,7 @@ GEM cocoon (1.2.15) coderay (1.1.3) color_diff (0.1) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.2) connection_pool (2.3.0) cose (1.2.1) cbor (~> 0.5.9) @@ -312,7 +312,7 @@ GEM httplog (1.6.0) rack (>= 2.0) rainbow (>= 2.0.0) - i18n (1.12.0) + i18n (1.14.1) concurrent-ruby (~> 1.0) i18n-tasks (1.0.12) activesupport (>= 4.0.2) @@ -401,9 +401,9 @@ GEM mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) - mini_mime (1.1.2) - mini_portile2 (2.8.2) - minitest (5.16.3) + mini_mime (1.1.5) + mini_portile2 (2.8.4) + minitest (5.20.0) msgpack (1.5.4) multi_json (1.15.0) multipart-post (2.1.1) @@ -412,7 +412,7 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-ssh (7.0.1) nio4r (2.5.9) - nokogiri (1.15.3) + nokogiri (1.15.4) mini_portile2 (~> 2.8.2) racc (~> 1.4) nsa (0.2.8) @@ -477,13 +477,13 @@ GEM pry-rails (0.3.9) pry (>= 0.10.4) public_suffix (5.0.0) - puma (5.6.5) + puma (5.6.7) nio4r (~> 2.0) pundit (2.2.0) activesupport (>= 3.0.0) raabro (1.4.0) racc (1.7.1) - rack (2.2.7) + rack (2.2.8) rack-attack (6.6.1) rack (>= 1.0, < 3) rack-cors (1.1.1) @@ -498,20 +498,20 @@ GEM rack rack-test (2.0.2) rack (>= 1.3) - rails (6.1.7.4) - actioncable (= 6.1.7.4) - actionmailbox (= 6.1.7.4) - actionmailer (= 6.1.7.4) - actionpack (= 6.1.7.4) - actiontext (= 6.1.7.4) - actionview (= 6.1.7.4) - activejob (= 6.1.7.4) - activemodel (= 6.1.7.4) - activerecord (= 6.1.7.4) - activestorage (= 6.1.7.4) - activesupport (= 6.1.7.4) + rails (6.1.7.6) + actioncable (= 6.1.7.6) + actionmailbox (= 6.1.7.6) + actionmailer (= 6.1.7.6) + actionpack (= 6.1.7.6) + actiontext (= 6.1.7.6) + actionview (= 6.1.7.6) + activejob (= 6.1.7.6) + activemodel (= 6.1.7.6) + activerecord (= 6.1.7.6) + activestorage (= 6.1.7.6) + activesupport (= 6.1.7.6) bundler (>= 1.15.0) - railties (= 6.1.7.4) + railties (= 6.1.7.6) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -527,9 +527,9 @@ GEM railties (>= 6.0.0, < 7) rails-settings-cached (0.6.6) rails (>= 4.2.0) - railties (6.1.7.4) - actionpack (= 6.1.7.4) - activesupport (= 6.1.7.4) + railties (6.1.7.6) + actionpack (= 6.1.7.6) + activesupport (= 6.1.7.6) method_source rake (>= 12.2) thor (~> 1.0) @@ -609,8 +609,8 @@ GEM activerecord (>= 4.0.0) railties (>= 4.0.0) semantic_range (3.0.0) - sidekiq (6.5.7) - connection_pool (>= 2.2.5) + sidekiq (6.5.11) + connection_pool (>= 2.2.5, < 3) rack (~> 2.0) redis (>= 4.5.0, < 5) sidekiq-bulk (0.2.0) @@ -718,14 +718,14 @@ GEM rack-proxy (>= 0.6.1) railties (>= 5.2) semantic_range (>= 2.3.0) - websocket-driver (0.7.5) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) wisper (2.0.1) xorcist (1.1.3) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.8) + zeitwerk (2.6.12) PLATFORMS ruby diff --git a/SECURITY.md b/SECURITY.md index ebb05ba2cc2c3b..5091fb9e60a471 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,6 +13,5 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through | Version | Supported | | ------- | ------------------ | | 4.1.x | Yes | -| 4.0.x | Until 2023-10-31 | -| 3.5.x | Until 2023-12-31 | -| < 3.5 | No | +| 4.0.x | No | +| < 4.0 | No | diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb index 2394574b3bede0..ecdccad4f7b96f 100644 --- a/app/controllers/concerns/signature_verification.rb +++ b/app/controllers/concerns/signature_verification.rb @@ -246,7 +246,7 @@ def actor_from_key_id(key_id) stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, ''), suppress_errors: false) } elsif !ActivityPub::TagManager.instance.local_uri?(key_id) account = ActivityPub::TagManager.instance.uri_to_actor(key_id) - account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false, suppress_errors: false) } + account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, suppress_errors: false) } account end rescue Mastodon::PrivateNetworkAddressError => e diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 102e4b13281ad7..18314d99f0a4a4 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -157,8 +157,8 @@ def safe_for_forwarding?(original, compacted) end end - def fetch_resource(uri, id, on_behalf_of = nil) - unless id + def fetch_resource(uri, id_is_known, on_behalf_of = nil) + unless id_is_known json = fetch_resource_without_id_validation(uri, on_behalf_of) return if !json.is_a?(Hash) || unsupported_uri_scheme?(json['id']) diff --git a/app/models/account_statuses_filter.rb b/app/lib/account_statuses_filter.rb similarity index 100% rename from app/models/account_statuses_filter.rb rename to app/lib/account_statuses_filter.rb diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index f4c67cccd733a7..e11ce49d97faaa 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -152,7 +152,9 @@ def follow_from_object def fetch_remote_original_status if object_uri.start_with?('http') return if ActivityPub::TagManager.instance.local_uri?(object_uri) - ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first) + + + ActivityPub::FetchRemoteStatusService.new.call(object_uri, on_behalf_of: @account.followers.local.first) elsif @object['url'].present? ::FetchRemoteStatusService.new.call(@object['url']) end diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb index f90adaf6c5fd3b..15c5d6bb8b9317 100644 --- a/app/lib/activitypub/linked_data_signature.rb +++ b/app/lib/activitypub/linked_data_signature.rb @@ -19,7 +19,7 @@ def verify_actor! return unless type == 'RsaSignature2017' creator = ActivityPub::TagManager.instance.uri_to_actor(creator_uri) - creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false) + creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri) return if creator.nil? diff --git a/app/lib/admin/account_statuses_filter.rb b/app/lib/admin/account_statuses_filter.rb new file mode 100644 index 00000000000000..94927e4b6806c9 --- /dev/null +++ b/app/lib/admin/account_statuses_filter.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class Admin::AccountStatusesFilter < AccountStatusesFilter + private + + def blocked? + false + end +end diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb index ea522c600cf2e4..7009db11f7bb26 100644 --- a/app/lib/importer/base_importer.rb +++ b/app/lib/importer/base_importer.rb @@ -34,7 +34,9 @@ def optimize_for_search! # Estimate the amount of documents that would be indexed. Not exact! # @returns [Integer] def estimate! - ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples AS estimate FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['estimate'].to_i } + reltuples = ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['reltuples'].to_i } + # If the table has never yet been vacuumed or analyzed, reltuples contains -1 + [reltuples, 0].max end # Import data from the database into the index diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb index a1d12a654eb43a..2e929d6e3f3f36 100644 --- a/app/lib/tag_manager.rb +++ b/app/lib/tag_manager.rb @@ -7,18 +7,18 @@ class TagManager include RoutingHelper def web_domain?(domain) - domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.web_domain).zero? + domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.web_domain).zero? end def local_domain?(domain) - domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.local_domain).zero? + domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.local_domain).zero? end def normalize_domain(domain) return if domain.nil? uri = Addressable::URI.new - uri.host = domain.gsub(/[\/]/, '') + uri.host = domain.delete_suffix('/') uri.normalized_host end @@ -28,7 +28,7 @@ def local_url?(url) domain = uri.host + (uri.port ? ":#{uri.port}" : '') TagManager.instance.web_domain?(domain) - rescue Addressable::URI::InvalidURIError + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError false end end diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index 537fd24c089662..2b4746a4d038be 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -46,7 +46,7 @@ def transform_response(str) raise UnexpectedResponseError unless json.is_a?(Hash) - Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com') + Translation.new(text: Sanitize.fragment(json.dig('translations', 0, 'text'), Sanitize::Config::MASTODON_STRICT), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com') rescue Oj::ParseError raise UnexpectedResponseError end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 43576e306274dc..ee668d298e0427 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -37,7 +37,7 @@ def transform_response(str, source_language) raise UnexpectedResponseError unless json.is_a?(Hash) - Translation.new(text: json['translatedText'], detected_source_language: source_language, provider: 'LibreTranslate') + Translation.new(text: Sanitize.fragment(json['translatedText'], Sanitize::Config::MASTODON_STRICT), detected_source_language: source_language, provider: 'LibreTranslate') rescue Oj::ParseError raise UnexpectedResponseError end diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb index 2896620cb21b09..f27d34868a2798 100644 --- a/app/lib/video_metadata_extractor.rb +++ b/app/lib/video_metadata_extractor.rb @@ -43,6 +43,9 @@ def parse_metadata @height = video_stream[:height] @frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate]) @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate]) + # For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we + # should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue. + @frame_rate ||= @r_frame_rate end if (audio_stream = audio_streams.first) diff --git a/app/models/account.rb b/app/models/account.rb index b3b425e0533854..f20d00caea30e8 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -61,9 +61,9 @@ class Account < ApplicationRecord trust_level ) - USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i - MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i - URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/ + USERNAME_RE = /[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?/i + MENTION_RE = %r{(? { without_unapproved.without_suspended.where(moved_to_account_id: nil) } scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) } scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) } - scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) } - scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) } + scope :by_recent_status, -> { includes(:account_stat).merge(AccountStat.order('last_status_at DESC NULLS LAST')).references(:account_stat) } + scope :by_recent_sign_in, -> { order(Arel.sql('users.current_sign_in_at DESC NULLS LAST')) } scope :popular, -> { order('account_stats.followers_count desc') } scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches("%.#{domain}"))) } scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) } diff --git a/app/models/admin/action_log_filter.rb b/app/models/admin/action_log_filter.rb index f89d452ef4f7bc..0117974628b7b7 100644 --- a/app/models/admin/action_log_filter.rb +++ b/app/models/admin/action_log_filter.rb @@ -38,7 +38,7 @@ class Admin::ActionLogFilter destroy_status: { target_type: 'Status', action: 'destroy' }.freeze, destroy_user_role: { target_type: 'UserRole', action: 'destroy' }.freeze, destroy_canonical_email_block: { target_type: 'CanonicalEmailBlock', action: 'destroy' }.freeze, - disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze, + disable_2fa_user: { target_type: 'User', action: 'disable_2fa' }.freeze, disable_custom_emoji: { target_type: 'CustomEmoji', action: 'disable' }.freeze, disable_user: { target_type: 'User', action: 'disable' }.freeze, enable_custom_emoji: { target_type: 'CustomEmoji', action: 'enable' }.freeze, diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb index 0f019b854d0741..0711d479c8c606 100644 --- a/app/models/admin/status_batch_action.rb +++ b/app/models/admin/status_batch_action.rb @@ -137,6 +137,6 @@ def report_params end def allowed_status_ids - AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id) + Admin::AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id) end end diff --git a/app/models/concerns/account_avatar.rb b/app/models/concerns/account_avatar.rb index e9b8b4adba23f2..b5919a9a23d585 100644 --- a/app/models/concerns/account_avatar.rb +++ b/app/models/concerns/account_avatar.rb @@ -18,7 +18,7 @@ def avatar_styles(file) included do # Avatar upload - has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail] + has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail] validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES validates_attachment_size :avatar, less_than: LIMIT remotable_attachment :avatar, LIMIT, suppress_errors: false diff --git a/app/models/concerns/account_header.rb b/app/models/concerns/account_header.rb index 0d197abfcd181d..e184880f93af39 100644 --- a/app/models/concerns/account_header.rb +++ b/app/models/concerns/account_header.rb @@ -19,7 +19,7 @@ def header_styles(file) included do # Header upload - has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail] + has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail] validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES validates_attachment_size :header, less_than: LIMIT remotable_attachment :header, LIMIT, suppress_errors: false diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index 3048056591fe29..4fae32c4869630 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -37,7 +37,7 @@ class CustomEmoji < ApplicationRecord belongs_to :category, class_name: 'CustomEmojiCategory', optional: true has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode - has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false + has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' } }, validate_media_type: false before_validation :downcase_domain diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index dbbd51a97ed826..f4921581a0e893 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -168,7 +168,7 @@ class MediaAttachment < ApplicationRecord }.freeze GLOBAL_CONVERT_OPTIONS = { - all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date', + all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp', }.freeze belongs_to :account, inverse_of: :media_attachments, optional: true diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index 56ca62d5ecd039..1fe71146900f8c 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -50,7 +50,7 @@ class PreviewCard < ApplicationRecord has_and_belongs_to_many :statuses has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy - has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date' }, validate_media_type: false + has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, validate_media_type: false validates :url, presence: true, uniqueness: true validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES diff --git a/app/models/preview_card_provider.rb b/app/models/preview_card_provider.rb index d61fe60208b44f..69ff4784e07d3c 100644 --- a/app/models/preview_card_provider.rb +++ b/app/models/preview_card_provider.rb @@ -25,7 +25,7 @@ class PreviewCardProvider < ApplicationRecord validates :domain, presence: true, uniqueness: true, domain: true - has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false + has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' } }, validate_media_type: false validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT } remotable_attachment :icon, LIMIT diff --git a/app/models/relationship_filter.rb b/app/models/relationship_filter.rb index 249fe3df8e1dfa..8e069c80a7e1fb 100644 --- a/app/models/relationship_filter.rb +++ b/app/models/relationship_filter.rb @@ -60,13 +60,13 @@ def scope_for(key, value) def relationship_scope(value) case value when 'following' - account.following.eager_load(:account_stat).reorder(nil) + account.following.includes(:account_stat).reorder(nil) when 'followed_by' - account.followers.eager_load(:account_stat).reorder(nil) + account.followers.includes(:account_stat).reorder(nil) when 'mutual' - account.followers.eager_load(:account_stat).reorder(nil).merge(Account.where(id: account.following)) + account.followers.includes(:account_stat).reorder(nil).merge(Account.where(id: account.following)) when 'invited' - Account.joins(user: :invite).merge(Invite.where(user: account.user)).eager_load(:account_stat).reorder(nil) + Account.joins(user: :invite).merge(Invite.where(user: account.user)).includes(:account_stat).reorder(nil) else raise Mastodon::InvalidParameterError, "Unknown relationship: #{value}" end @@ -112,7 +112,7 @@ def order_scope(value) def activity_scope(value) case value when 'dormant' - AccountStat.where(last_status_at: nil).or(AccountStat.where(AccountStat.arel_table[:last_status_at].lt(1.month.ago))) + Account.joins(:account_stat).where(account_stat: { last_status_at: [nil, ...1.month.ago] }) else raise Mastodon::InvalidParameterError, "Unknown activity: #{value}" end diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb index 167131fdd9ef00..c2167070694e10 100644 --- a/app/models/site_upload.rb +++ b/app/models/site_upload.rb @@ -40,7 +40,7 @@ class SiteUpload < ApplicationRecord mascot: {}.freeze, }.freeze - has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] + has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/ validates :file, presence: true diff --git a/app/models/status.rb b/app/models/status.rb index 4381eaec771671..7b6c4eca63b3e2 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -354,13 +354,25 @@ def reload_stale_associations!(cached_items) account_ids.uniq! + status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq + return if account_ids.empty? accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id) + status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id) + cached_items.each do |item| item.account = accounts[item.account_id] item.reblog.account = accounts[item.reblog.account_id] if item.reblog? + + if item.reblog? + status_stat = status_stats[item.reblog.id] + item.reblog.status_stat = status_stat if status_stat.present? + else + status_stat = status_stats[item.id] + item.status_stat = status_stat if status_stat.present? + end end end diff --git a/app/policies/admin/status_policy.rb b/app/policies/admin/status_policy.rb index ffaa30f13de60a..e9379c25eca903 100644 --- a/app/policies/admin/status_policy.rb +++ b/app/policies/admin/status_policy.rb @@ -12,7 +12,7 @@ def index? end def show? - role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported?) + role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported? || viewable_through_normal_policy?) end def destroy? @@ -26,4 +26,10 @@ def update? def review? role.can?(:manage_taxonomies) end + + private + + def viewable_through_normal_policy? + StatusPolicy.new(current_account, record, @preloaded_relations).show? + end end diff --git a/app/policies/backup_policy.rb b/app/policies/backup_policy.rb index 0ef89a8d0c8fb9..86b8efbe96fa7d 100644 --- a/app/policies/backup_policy.rb +++ b/app/policies/backup_policy.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class BackupPolicy < ApplicationPolicy - MIN_AGE = 1.week + MIN_AGE = 6.days def create? user_signed_in? && current_user.backups.where('created_at >= ?', MIN_AGE.ago).count.zero? diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index ca7a8c6ca8930b..744228645e6b78 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -2,7 +2,7 @@ class ActivityPub::FetchRemoteAccountService < ActivityPub::FetchRemoteActorService # Does a WebFinger roundtrip on each call, unless `only_key` is true - def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true) + def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true) actor = super return actor if actor.nil? || actor.is_a?(Account) diff --git a/app/services/activitypub/fetch_remote_actor_service.rb b/app/services/activitypub/fetch_remote_actor_service.rb index db09c38d82697b..bab2965ec45e53 100644 --- a/app/services/activitypub/fetch_remote_actor_service.rb +++ b/app/services/activitypub/fetch_remote_actor_service.rb @@ -10,15 +10,15 @@ class Error < StandardError; end SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze # Does a WebFinger roundtrip on each call, unless `only_key` is true - def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true) + def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true) return if domain_not_allowed?(uri) return ActivityPub::TagManager.instance.uri_to_actor(uri) if ActivityPub::TagManager.instance.local_uri?(uri) @json = begin if prefetched_body.nil? - fetch_resource(uri, id) + fetch_resource(uri, true) else - body_to_json(prefetched_body, compare_id: id ? uri : nil) + body_to_json(prefetched_body, compare_id: uri) end rescue Oj::ParseError raise Error, "Error parsing JSON-LD document #{uri}" diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb index 32e82b47a58d67..1f671849236cd8 100644 --- a/app/services/activitypub/fetch_remote_key_service.rb +++ b/app/services/activitypub/fetch_remote_key_service.rb @@ -6,23 +6,10 @@ class ActivityPub::FetchRemoteKeyService < BaseService class Error < StandardError; end # Returns actor that owns the key - def call(uri, id: true, prefetched_body: nil, suppress_errors: true) + def call(uri, suppress_errors: true) raise Error, 'No key URI given' if uri.blank? - if prefetched_body.nil? - if id - @json = fetch_resource_without_id_validation(uri) - if actor_type? - @json = fetch_resource(@json['id'], true) - elsif uri != @json['id'] - raise Error, "Fetched URI #{uri} has wrong id #{@json['id']}" - end - else - @json = fetch_resource(uri, id) - end - else - @json = body_to_json(prefetched_body, compare_id: id ? uri : nil) - end + @json = fetch_resource(uri, false) raise Error, "Unable to fetch key JSON at #{uri}" if @json.nil? raise Error, "Unsupported JSON-LD context for document #{uri}" unless supported_context?(@json) diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 80309824509a47..d9227d300ef298 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -4,12 +4,12 @@ class ActivityPub::FetchRemoteStatusService < BaseService include JsonLdHelper # Should be called when uri has already been checked for locality - def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil) + def call(uri, prefetched_body: nil, on_behalf_of: nil) @json = begin if prefetched_body.nil? - fetch_resource(uri, id, on_behalf_of) + fetch_resource(uri, true, on_behalf_of) else - body_to_json(prefetched_body, compare_id: id ? uri : nil) + body_to_json(prefetched_body, compare_id: uri) end end @@ -52,7 +52,7 @@ def trustworthy_attribution?(uri, attributed_to) def account_from_uri(uri) actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account) - actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true) if actor.nil? || actor.possibly_stale? + actor = ActivityPub::FetchRemoteAccountService.new.call(uri) if actor.nil? || actor.possibly_stale? actor end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 16731c5146fb53..afcb1222a92c48 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -257,7 +257,7 @@ def collection_info(type) def moved_account account = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account) - account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true) + account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], break_on_redirect: true) account end diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb index 73204e55db0c41..e1af2347852111 100644 --- a/app/services/fetch_resource_service.rb +++ b/app/services/fetch_resource_service.rb @@ -47,7 +47,15 @@ def process_response(response, terminal = false) body = response.body_with_limit json = body_to_json(body) - [json['id'], { prefetched_body: body, id: true }] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json)) + return unless supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json)) + + if json['id'] != @url + return if terminal + + return process(json['id'], terminal: true) + end + + [@url, { prefetched_body: body }] elsif !terminal link_header = response['Link'] && parse_link_header(response) diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index 539a0d9db5fd93..6e6ed87b0a31da 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -12,7 +12,9 @@ def call(status, target_language) @content = status_content_format(@status) @target_language = target_language - Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@content, @status.language, @target_language) } + Rails.cache.fetch("translations:v2/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do + translation_backend.translate(@content, @status.language, @target_language) + end end private diff --git a/app/workers/account_deletion_worker.rb b/app/workers/account_deletion_worker.rb index fdf013e01043b1..7b8a31f8c6c52f 100644 --- a/app/workers/account_deletion_worker.rb +++ b/app/workers/account_deletion_worker.rb @@ -3,7 +3,7 @@ class AccountDeletionWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', lock: :until_executed + sidekiq_options queue: 'pull', lock: :until_executed, lock_ttl: 1.week.to_i def perform(account_id, options = {}) reserve_username = options.with_indifferent_access.fetch(:reserve_username, true) diff --git a/app/workers/activitypub/synchronize_featured_collection_worker.rb b/app/workers/activitypub/synchronize_featured_collection_worker.rb index f67d693cb3ab3c..7a187d7f53eede 100644 --- a/app/workers/activitypub/synchronize_featured_collection_worker.rb +++ b/app/workers/activitypub/synchronize_featured_collection_worker.rb @@ -3,7 +3,7 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', lock: :until_executed + sidekiq_options queue: 'pull', lock: :until_executed, lock_ttl: 1.day.to_i def perform(account_id, options = {}) options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys) diff --git a/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb index 14af4f725cdd62..570415c82149c5 100644 --- a/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb +++ b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb @@ -3,7 +3,7 @@ class ActivityPub::SynchronizeFeaturedTagsCollectionWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', lock: :until_executed + sidekiq_options queue: 'pull', lock: :until_executed, lock_ttl: 1.day.to_i def perform(account_id, url) ActivityPub::FetchFeaturedTagsCollectionService.new.call(Account.find(account_id), url) diff --git a/app/workers/activitypub/update_distribution_worker.rb b/app/workers/activitypub/update_distribution_worker.rb index d0391bb6f61169..a04ac621f30e02 100644 --- a/app/workers/activitypub/update_distribution_worker.rb +++ b/app/workers/activitypub/update_distribution_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class ActivityPub::UpdateDistributionWorker < ActivityPub::RawDistributionWorker - sidekiq_options queue: 'push', lock: :until_executed + sidekiq_options queue: 'push', lock: :until_executed, lock_ttl: 1.day.to_i # Distribute an profile update to servers that might have a copy # of the account in question diff --git a/app/workers/admin/account_deletion_worker.rb b/app/workers/admin/account_deletion_worker.rb index 6e0eb331bef83d..5dfdfb6e73c14d 100644 --- a/app/workers/admin/account_deletion_worker.rb +++ b/app/workers/admin/account_deletion_worker.rb @@ -3,7 +3,7 @@ class Admin::AccountDeletionWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', lock: :until_executed + sidekiq_options queue: 'pull', lock: :until_executed, lock_ttl: 1.week.to_i def perform(account_id) DeleteAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: true) diff --git a/app/workers/admin/domain_purge_worker.rb b/app/workers/admin/domain_purge_worker.rb index 095232a6d74af1..6c5250b660c380 100644 --- a/app/workers/admin/domain_purge_worker.rb +++ b/app/workers/admin/domain_purge_worker.rb @@ -3,7 +3,7 @@ class Admin::DomainPurgeWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', lock: :until_executed + sidekiq_options queue: 'pull', lock: :until_executed, lock_ttl: 1.week.to_i def perform(domain) PurgeDomainService.new.call(domain) diff --git a/app/workers/publish_scheduled_status_worker.rb b/app/workers/publish_scheduled_status_worker.rb index ce42f7be7c6e21..aa5c4a834a0510 100644 --- a/app/workers/publish_scheduled_status_worker.rb +++ b/app/workers/publish_scheduled_status_worker.rb @@ -3,7 +3,7 @@ class PublishScheduledStatusWorker include Sidekiq::Worker - sidekiq_options lock: :until_executed + sidekiq_options lock: :until_executed, lock_ttl: 1.hour.to_i def perform(scheduled_status_id) scheduled_status = ScheduledStatus.find(scheduled_status_id) diff --git a/app/workers/resolve_account_worker.rb b/app/workers/resolve_account_worker.rb index 2b5be6d1b217da..4ae2442af52e02 100644 --- a/app/workers/resolve_account_worker.rb +++ b/app/workers/resolve_account_worker.rb @@ -3,7 +3,7 @@ class ResolveAccountWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', lock: :until_executed + sidekiq_options queue: 'pull', lock: :until_executed, lock_ttl: 1.day.to_i def perform(uri) ResolveAccountService.new.call(uri) diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index d622f5586e4666..cde6210fbaf97c 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -4,7 +4,7 @@ class Scheduler::IndexingScheduler include Sidekiq::Worker include Redisable - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 30.minutes.to_i IMPORT_BATCH_SIZE = 1000 SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE diff --git a/app/workers/scheduler/scheduled_statuses_scheduler.rb b/app/workers/scheduler/scheduled_statuses_scheduler.rb index 3bf6300b3c4b69..fe60d5524eaf20 100644 --- a/app/workers/scheduler/scheduled_statuses_scheduler.rb +++ b/app/workers/scheduler/scheduled_statuses_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::ScheduledStatusesScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.hour.to_i def perform publish_scheduled_statuses! diff --git a/app/workers/scheduler/trends/refresh_scheduler.rb b/app/workers/scheduler/trends/refresh_scheduler.rb index b559ba46b4b521..85c000deea786d 100644 --- a/app/workers/scheduler/trends/refresh_scheduler.rb +++ b/app/workers/scheduler/trends/refresh_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::Trends::RefreshScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 30.minutes.to_i def perform Trends.refresh! diff --git a/app/workers/verify_account_links_worker.rb b/app/workers/verify_account_links_worker.rb index f606e6c26fefdf..ad27f450b78996 100644 --- a/app/workers/verify_account_links_worker.rb +++ b/app/workers/verify_account_links_worker.rb @@ -3,7 +3,7 @@ class VerifyAccountLinksWorker include Sidekiq::Worker - sidekiq_options queue: 'default', retry: false, lock: :until_executed + sidekiq_options queue: 'default', retry: false, lock: :until_executed, lock_ttl: 1.hour.to_i def perform(account_id) account = Account.find(account_id) diff --git a/docker-compose.yml b/docker-compose.yml index 2efd47d9783ad7..513c5dea73cc57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.0.9 + image: ghcr.io/mastodon/mastodon:v4.0.13 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.0.9 + image: ghcr.io/mastodon/mastodon:v4.0.13 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.0.9 + image: ghcr.io/mastodon/mastodon:v4.0.13 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 873969cab2e672..ca2966badc7d49 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ def minor end def patch - 9 + 13 end def flags diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index be40b4924107aa..0f2e30f7d5e45e 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -37,12 +37,14 @@ def make @output_options['f'] = 'image2' @output_options['vframes'] = 1 when 'mp4' - @output_options['acodec'] = 'aac' - @output_options['strict'] = 'experimental' - - if high_vfr?(metadata) && !eligible_to_passthrough?(metadata) - @output_options['vsync'] = 'vfr' - @output_options['r'] = @vfr_threshold + unless eligible_to_passthrough?(metadata) + @output_options['acodec'] = 'aac' + @output_options['strict'] = 'experimental' + + if high_vfr?(metadata) + @output_options['vsync'] = 'vfr' + @output_options['r'] = @vfr_threshold + end end end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 227688e23643c3..877c7e63ebfb30 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -40,24 +40,36 @@ end describe 'POST #batch' do - before do - post :batch, params: { :account_id => account.id, action => '', :admin_status_batch_action => { status_ids: status_ids } } - end + subject { post :batch, params: { :account_id => account.id, action => '', :admin_status_batch_action => { status_ids: status_ids } } } let(:status_ids) { [media_attached_status.id] } - context 'when action is report' do + shared_examples 'when action is report' do let(:action) { 'report' } it 'creates a report' do + subject + report = Report.last expect(report.target_account_id).to eq account.id expect(report.status_ids).to eq status_ids end it 'redirects to report page' do + subject + expect(response).to redirect_to(admin_report_path(Report.last.id)) end end + + it_behaves_like 'when action is report' + + context 'when the moderator is blocked by the author' do + before do + account.block!(user.account) + end + + it_behaves_like 'when action is report' + end end end diff --git a/spec/controllers/concerns/cache_concern_spec.rb b/spec/controllers/concerns/cache_concern_spec.rb index a34d7d72676964..21daa19921007e 100644 --- a/spec/controllers/concerns/cache_concern_spec.rb +++ b/spec/controllers/concerns/cache_concern_spec.rb @@ -13,12 +13,17 @@ def empty_array def empty_relation render plain: cache_collection(Status.none, Status).size end + + def account_statuses_favourites + render plain: cache_collection(Status.where(account_id: params[:id]), Status).map(&:favourites_count) + end end before do routes.draw do - get 'empty_array' => 'anonymous#empty_array' - post 'empty_relation' => 'anonymous#empty_relation' + get 'empty_array' => 'anonymous#empty_array' + get 'empty_relation' => 'anonymous#empty_relation' + get 'account_statuses_favourites' => 'anonymous#account_statuses_favourites' end end @@ -36,5 +41,20 @@ def empty_relation expect(response.body).to eq '0' end end + + context 'when given a collection of statuses' do + let!(:account) { Fabricate(:account) } + let!(:status) { Fabricate(:status, account: account) } + + it 'correctly updates with new interactions' do + get :account_statuses_favourites, params: { id: account.id } + expect(response.body).to eq '[0]' + + FavouriteService.new.call(account, status) + + get :account_statuses_favourites, params: { id: account.id } + expect(response.body).to eq '[1]' + end + end end end diff --git a/spec/fabricators/account_stat_fabricator.rb b/spec/fabricators/account_stat_fabricator.rb index 2b06b4790920de..20272fb22f202d 100644 --- a/spec/fabricators/account_stat_fabricator.rb +++ b/spec/fabricators/account_stat_fabricator.rb @@ -1,6 +1,8 @@ +# frozen_string_literal: true + Fabricator(:account_stat) do - account nil - statuses_count "" - following_count "" - followers_count "" + account { Fabricate.build(:account) } + statuses_count '123' + following_count '456' + followers_count '789' end diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index edae05f9db6d86..a40dfae96f37ca 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -689,7 +689,7 @@ expect(subject.match('Check this out https://medium.com/@alice/some-article#.abcdef123')).to be_nil end - xit 'does not match URL querystring' do + it 'does not match URL query string' do expect(subject.match('https://example.com/?x=@alice')).to be_nil end end diff --git a/spec/models/relationship_filter_spec.rb b/spec/models/relationship_filter_spec.rb index 7c0f37a06f299e..fccd42aaad0622 100644 --- a/spec/models/relationship_filter_spec.rb +++ b/spec/models/relationship_filter_spec.rb @@ -6,32 +6,60 @@ let(:account) { Fabricate(:account) } describe '#results' do - context 'when default params are used' do - let(:subject) do - RelationshipFilter.new(account, 'order' => 'active').results - end + let(:account_of_7_months) { Fabricate(:account_stat, statuses_count: 1, last_status_at: 7.months.ago).account } + let(:account_of_1_day) { Fabricate(:account_stat, statuses_count: 1, last_status_at: 1.day.ago).account } + let(:account_of_3_days) { Fabricate(:account_stat, statuses_count: 1, last_status_at: 3.days.ago).account } + let(:silent_account) { Fabricate(:account_stat, statuses_count: 0, last_status_at: nil).account } + + before do + account.follow!(account_of_7_months) + account.follow!(account_of_1_day) + account.follow!(account_of_3_days) + account.follow!(silent_account) + end - before do - add_following_account_with(last_status_at: 7.days.ago) - add_following_account_with(last_status_at: 1.day.ago) - add_following_account_with(last_status_at: 3.days.ago) + context 'when ordering by last activity' do + context 'when not filtering' do + subject do + described_class.new(account, 'order' => 'active').results + end + + it 'returns followings ordered by last activity' do + expect(subject).to eq [account_of_1_day, account_of_3_days, account_of_7_months, silent_account] + end end - it 'returns followings ordered by last activity' do - expected_result = account.following.eager_load(:account_stat).reorder(nil).by_recent_status + context 'when filtering for dormant accounts' do + subject do + described_class.new(account, 'order' => 'active', 'activity' => 'dormant').results + end - expect(subject).to eq expected_result + it 'returns dormant followings ordered by last activity' do + expect(subject).to eq [account_of_7_months, silent_account] + end end end - end - def add_following_account_with(last_status_at:) - following_account = Fabricate(:account) - Fabricate(:account_stat, account: following_account, - last_status_at: last_status_at, - statuses_count: 1, - following_count: 0, - followers_count: 0) - Fabricate(:follow, account: account, target_account: following_account).account + context 'when ordering by account creation' do + context 'when not filtering' do + subject do + described_class.new(account, 'order' => 'recent').results + end + + it 'returns followings ordered by last account creation' do + expect(subject).to eq [silent_account, account_of_3_days, account_of_1_day, account_of_7_months] + end + end + + context 'when filtering for dormant accounts' do + subject do + described_class.new(account, 'order' => 'recent', 'activity' => 'dormant').results + end + + it 'returns dormant followings ordered by last activity' do + expect(subject).to eq [silent_account, account_of_7_months] + end + end + end end end diff --git a/spec/services/activitypub/fetch_remote_account_service_spec.rb b/spec/services/activitypub/fetch_remote_account_service_spec.rb index ec6f1f41d8f6ce..9ee2c15248069d 100644 --- a/spec/services/activitypub/fetch_remote_account_service_spec.rb +++ b/spec/services/activitypub/fetch_remote_account_service_spec.rb @@ -16,7 +16,7 @@ end describe '#call' do - let(:account) { subject.call('https://example.com/alice', id: true) } + let(:account) { subject.call('https://example.com/alice') } shared_examples 'sets profile data' do it 'returns an account' do diff --git a/spec/services/activitypub/fetch_remote_actor_service_spec.rb b/spec/services/activitypub/fetch_remote_actor_service_spec.rb index 20117c66d04764..56805b33254760 100644 --- a/spec/services/activitypub/fetch_remote_actor_service_spec.rb +++ b/spec/services/activitypub/fetch_remote_actor_service_spec.rb @@ -16,7 +16,7 @@ end describe '#call' do - let(:account) { subject.call('https://example.com/alice', id: true) } + let(:account) { subject.call('https://example.com/alice') } shared_examples 'sets profile data' do it 'returns an account' do diff --git a/spec/services/activitypub/fetch_remote_key_service_spec.rb b/spec/services/activitypub/fetch_remote_key_service_spec.rb index 3186c4270d7e3d..9c818d12c0d634 100644 --- a/spec/services/activitypub/fetch_remote_key_service_spec.rb +++ b/spec/services/activitypub/fetch_remote_key_service_spec.rb @@ -43,7 +43,7 @@ end describe '#call' do - let(:account) { subject.call(public_key_id, id: false) } + let(:account) { subject.call(public_key_id) } context 'when the key is a sub-object from the actor' do before do diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb index c0c96ab69c95f6..412c410575fd38 100644 --- a/spec/services/fetch_resource_service_spec.rb +++ b/spec/services/fetch_resource_service_spec.rb @@ -54,7 +54,7 @@ let(:json) do { - id: 1, + id: 'http://example.com/foo', '@context': ActivityPub::TagManager::CONTEXT, type: 'Note', }.to_json @@ -79,14 +79,14 @@ let(:content_type) { 'application/activity+json; charset=utf-8' } let(:body) { json } - it { is_expected.to eq [1, { prefetched_body: body, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: body }] } end context 'when content type is ld+json with profile' do let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' } let(:body) { json } - it { is_expected.to eq [1, { prefetched_body: body, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: body }] } end before do @@ -97,14 +97,14 @@ context 'when link header is present' do let(:headers) { { 'Link' => '; rel="alternate"; type="application/activity+json"', } } - it { is_expected.to eq [1, { prefetched_body: json, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: json }] } end context 'when content type is text/html' do let(:content_type) { 'text/html' } let(:body) { '' } - it { is_expected.to eq [1, { prefetched_body: json, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: json }] } end end end diff --git a/spec/services/resolve_url_service_spec.rb b/spec/services/resolve_url_service_spec.rb index ab5b50b765058d..85a672524878ff 100644 --- a/spec/services/resolve_url_service_spec.rb +++ b/spec/services/resolve_url_service_spec.rb @@ -139,6 +139,7 @@ stub_request(:get, url).to_return(status: 302, headers: { 'Location' => status_url }) body = ActiveModelSerializers::SerializableResource.new(status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter).to_json stub_request(:get, status_url).to_return(body: body, headers: { 'Content-Type' => 'application/activity+json' }) + stub_request(:get, uri).to_return(body: body, headers: { 'Content-Type' => 'application/activity+json' }) end it 'returns status by url' do