diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 76d2de4c19..c123319c2a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -17,7 +17,7 @@ jobs: eb-environment-name: ${{ steps.output-app-env.outputs.eb-environment-name }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set env to staging id: set-env-staging @@ -45,7 +45,7 @@ jobs: needs: create-app-env steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 @@ -77,7 +77,7 @@ jobs: - build steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 10d4e0ac52..d6a768b152 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - name: apt-get diff --git a/.github/workflows/rails-test.yml b/.github/workflows/rails-test.yml index afccf7f6e7..eb809ee0ad 100644 --- a/.github/workflows/rails-test.yml +++ b/.github/workflows/rails-test.yml @@ -17,6 +17,7 @@ jobs: DATABASE_USERNAME: postgres DATABASE_PASSWORD: postgres RAILS_ENV: test + IMAGEMAGICK_SRC: 7.1.0-50.tar.gz services: db: @@ -37,13 +38,43 @@ jobs: - 6379/tcp steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - name: apt-get run: | sudo apt-get update -y sudo apt-get -yqq install libpq-dev postgresql-client + + - name: remove imagemagick 6 + run: sudo apt remove imagemagick + - name: setup imagemagick 7 + run: | + sudo sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list + sudo apt update + sudo apt install build-essential + sudo apt build-dep imagemagick + sudo apt install libwebp-dev libopenjp2-7-dev librsvg2-dev libde265-dev + - name: install libheif + run: | + git clone https://github.com/strukturag/libheif.git + cd libheif + ./autogen.sh + ./configure --disable-go --disable-examples + make + sudo make install + - name: install imagemagick 7 + run: | + wget https://github.com/ImageMagick/ImageMagick/archive/refs/tags/${IMAGEMAGICK_SRC} + tar xf ${IMAGEMAGICK_SRC} + cd ImageMagick-7* + ./configure + make + sudo make install + sudo ldconfig + - name: check imagemagick + run: convert -version + - name: Set up Ruby 2.7 uses: ruby/setup-ruby@v1 with: @@ -51,7 +82,7 @@ jobs: bundler-cache: true - name: setup Node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: 16.9.1 cache: 'yarn' diff --git a/Gemfile b/Gemfile index 82f69853d5..360e4f1c5a 100644 --- a/Gemfile +++ b/Gemfile @@ -8,13 +8,15 @@ gem "decidim", "0.25.2" gem "decidim-comments", path: "decidim-comments" -gem "decidim-decidim_awesome", "~> 0.8.1" +gem "decidim-decidim_awesome", git: "https://github.com/codeforjapan/decidim-module-decidim_awesome.git", branch: "v0.8.3-2022-10-20" gem "decidim-term_customizer", git: "https://github.com/codeforjapan/decidim-module-term_customizer.git", branch: "025-ja" gem "bootsnap" gem "puma", ">= 5.0.0" +gem "puma_worker_killer" + gem "uglifier", "~> 4.1" gem "faker", "~> 2.14" @@ -22,6 +24,7 @@ gem "faker", "~> 2.14" gem "wicked_pdf", "~> 2.1" gem "deface" +gem "image_processing" gem "newrelic_rpm" gem "omniauth-line_login", path: "omniauth-line_login" diff --git a/Gemfile.lock b/Gemfile.lock index c7d04cc600..5475459342 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,13 @@ +GIT + remote: https://github.com/codeforjapan/decidim-module-decidim_awesome.git + revision: 658d226e9ed164ffdca2efbde4dfaf79b02c295e + branch: v0.8.3-2022-10-20 + specs: + decidim-decidim_awesome (0.8.3) + decidim-admin (>= 0.25.0, < 0.27) + decidim-core (>= 0.25.0, < 0.27) + sassc (~> 2.3) + GIT remote: https://github.com/codeforjapan/decidim-module-term_customizer.git revision: a0ccb0b32d6d7943af90ff4f8029fd4e7b2afa8f @@ -293,10 +303,6 @@ GEM decidim-debates (0.25.2) decidim-comments (= 0.25.2) decidim-core (= 0.25.2) - decidim-decidim_awesome (0.8.2) - decidim-admin (>= 0.25.0, < 0.27) - decidim-core (>= 0.25.0, < 0.27) - sassc (~> 2.3) decidim-dev (0.25.2) axe-core-rspec (~> 4.1.0) byebug (~> 11.0) @@ -459,6 +465,8 @@ GEM activesupport (>= 4.1, < 7.1) railties (>= 4.1, < 7.1) geocoder (1.7.5) + get_process_mem (0.2.7) + ffi (~> 1.0) globalid (1.0.0) activesupport (>= 5.0) graphql (1.13.12) @@ -610,6 +618,9 @@ GEM public_suffix (4.0.7) puma (5.6.4) nio4r (~> 2.0) + puma_worker_killer (0.3.1) + get_process_mem (~> 0.2) + puma (>= 2.7) racc (1.6.0) rack (2.2.3) rack-attack (6.6.1) @@ -852,7 +863,7 @@ DEPENDENCIES byebug (~> 11.0) decidim (= 0.25.2) decidim-comments! - decidim-decidim_awesome (~> 0.8.1) + decidim-decidim_awesome! decidim-dev (= 0.25.2) decidim-term_customizer! decidim-user_extension! @@ -862,12 +873,14 @@ DEPENDENCIES faker (~> 2.14) figaro fog-aws + image_processing letter_opener_web (~> 1.3) listen (~> 3.1) newrelic_rpm omniauth-line_login! omniauth-rails_csrf_protection puma (>= 5.0.0) + puma_worker_killer rspec-rails rubocop-faker rubyzip (>= 1.0.0) diff --git a/app/packs/src/decidim/decidim_awesome/editors/editor.js b/app/packs/src/decidim/decidim_awesome/editors/editor.js index 5afe420f4a..f6c6447126 100644 --- a/app/packs/src/decidim/decidim_awesome/editors/editor.js +++ b/app/packs/src/decidim/decidim_awesome/editors/editor.js @@ -355,7 +355,7 @@ export function createQuillEditor(container) { modules: ["Resize", "DisplaySize"] } modules.imageUpload = { - url: $(container).data("uploadImagesPath") || DecidimAwesome.editor_uploader_path, + url: DecidimAwesome.editor_uploader_path, method: "POST", name: "image", withCredentials: false, @@ -366,7 +366,18 @@ export function createQuillEditor(container) { }, callbackKO: (serverError) => { $("div.ql-toolbar").last().removeClass("editor-loading") - console.error(`Image upload error: ${serverError.message}`); + let msg = serverError && serverError.body; + try { + msg = JSON.parse(msg).message; + } catch (e) { console.error("Parsing error", e); } + console.error(`Image upload error: ${msg}`); + let $p = $(`

${msg}

`); + $(container).after($p) + setTimeout(() => { + $p.fadeOut(1000, () => { + $p.destroy(); + }); + }, 3000); }, checkBeforeSend: (file, next) => { $("div.ql-toolbar").last().addClass("editor-loading") diff --git a/app/previewers/heic_previewer.rb b/app/previewers/heic_previewer.rb new file mode 100644 index 0000000000..5b3fd36959 --- /dev/null +++ b/app/previewers/heic_previewer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class HeicPreviewer < ActiveStorage::Previewer + CONTENT_TYPES = %w(image/heic image/heif image/heic-sequence image/heif-sequence).freeze + + class << self + def accept?(blob) + CONTENT_TYPES.include?(blob.content_type) && minimagick_exists? + end + + def minimagick_exists? + return @minimagick_exists if @minimagick_exists.present? + + @minimagick_exists = defined?(ImageProcessing::MiniMagick) + Rails.logger.error "#{self.class} :: MiniMagick is not installed" unless @minimagick_exists + + @minimagick_exists + end + end + + def preview(**_options) + download_blob_to_tempfile do |input| + begin + io = ImageProcessing::MiniMagick.source(input).convert("png").call + rescue ImageProcessing::Error + io = ImageProcessing::MiniMagick.loader(page: 0).source(input).convert("png").call + end + yield io: io, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end +end diff --git a/config/initializers/active_storage.rb b/config/initializers/active_storage.rb new file mode 100644 index 0000000000..03ea050f60 --- /dev/null +++ b/config/initializers/active_storage.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +Rails.application.configure do + config.active_storage.previewers << HeicPreviewer + config.active_storage.variable_content_types << "image/heic" + config.active_storage.variable_content_types << "image/heif" +end diff --git a/config/initializers/decidim.rb b/config/initializers/decidim.rb index 57d77087cc..b81ef1ff23 100644 --- a/config/initializers/decidim.rb +++ b/config/initializers/decidim.rb @@ -242,4 +242,6 @@ Decidim.register_assets_path File.expand_path("app/packs", Rails.application.root) # Set max_complexity of GraphQL::Schema -Decidim::Api::Schema.max_complexity = 100_000 +Rails.application.config.to_prepare do + Decidim::Api::Schema.max_complexity = 100_000 +end diff --git a/config/puma.rb b/config/puma.rb index 78638e5e45..3fd08a36ee 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -20,13 +20,25 @@ # Specifies the `pidfile` that Puma will use. pidfile ENV.fetch("PIDFILE", "tmp/pids/server.pid") +before_fork do + PumaWorkerKiller.config do |config| + config.ram = 2048 + config.frequency = 60 + config.percent_usage = 0.9 + config.rolling_restart_frequency = 24 * 60 * 60 + config.reaper_status_logs = true + config.pre_term = ->(worker) { puts "Worker #{worker.index}(#{worker.pid}) being killed" } + end + PumaWorkerKiller.start +end + # Specifies the number of `workers` to boot in clustered mode. # Workers are forked webserver processes. If using threads and workers together # the concurrency of the application would be max `threads` * `workers`. # Workers do not work on JRuby or Windows (both of which do not support # processes). # -# workers ENV.fetch("WEB_CONCURRENCY") { 2 } +workers ENV.fetch("WEB_CONCURRENCY", 2) # Use the `preload_app!` method when specifying a `workers` number. # This directive tells Puma to first boot the application and load code diff --git a/docker-compose-etherpad.yml b/docker-compose-etherpad.yml deleted file mode 100644 index f5b37baabe..0000000000 --- a/docker-compose-etherpad.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Please replace all the values that start with CHANGE_ME. -version: '3' -services: - app: - environment: - # Title of the Etherpad Lite instance. Defaults to "Etherpad". - - ETHERPAD_TITLE=Decidim pads - # Port of the Etherpad Lite instance. Defaults to 9001. - - ETHERPAD_PORT=9001 - # If set, an admin account is enabled for Etherpad, and the /admin/ interface is accessible via it. - - ETHERPAD_ADMIN_PASSWORD=CHANGE_ME_ADMIN_PASSWORD - # If the admin password is set, this defaults to "admin". Otherwise the user can set it to another username. - - ETHERPAD_ADMIN_USER=CHANGE_ME_ADMIN_USER - # Type of databse to use. Defaults to mysql. - - ETHERPAD_DB_TYPE=mysql - # Hostname of the database to use. Defaults to mysql. - - ETHERPAD_DB_HOST=mysql - # By default Etherpad Lite will attempt to connect as root to the database container. - - ETHERPAD_DB_USER=root - # Password to use, mandatory. If legacy links are used and ETHERPAD_DB_USER is root, then MYSQL_ENV_MYSQL_ROOT_PASSWORD is automatically used. - - ETHERPAD_DB_PASSWORD=CHANGE_ME_PASSWORD - # The database to use. Defaults to etherpad. If the database is not available, it will be created when the container is launched (only if the database type is either mysql or postgres, and the user need to have the right to create the database). - - ETHERPAD_DB_NAME=etherpad - # The charset to use. Defaults to utf8mb4. - - ETHERPAD_DB_CHARSET=utf8mb4 - # if file APIKEY.txt is missing, the variable value is used to provision it - - ETHERPAD_API_KEY=CHANGE_ME_API_KEY - image: 'tvelocity/etherpad-lite' - deploy: - replicas: 1 - update_config: - parallelism: 1 - restart_policy: - condition: on-failure - labels: - - com.df.notify=true - - com.df.port=9001 - - com.df.serviceDomain=CHANGE_ME_PADS_SERVER_HOST_NAME - proxy: - image: dockerflow/docker-flow-proxy - ports: - - 80:80 - - 443:443 - environment: - - LISTENER_ADDRESS=swarm-listener - - MODE=swarm - deploy: - replicas: 1 - restart_policy: - condition: on-failure - swarm-listener: - image: dockerflow/docker-flow-swarm-listener - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - - DF_NOTIFY_CREATE_SERVICE_URL=http://proxy:8080/v1/docker-flow-proxy/reconfigure - - DF_NOTIFY_REMOVE_SERVICE_URL=http://proxy:8080/v1/docker-flow-proxy/remove - deploy: - replicas: 1 - restart_policy: - condition: on-failure - mysql: - environment: - - MYSQL_ROOT_PASSWORD=CHANGE_ME_PASSWORD - image: 'mysql:5.7' - volumes: - - mysql:/var/lib/mysql - deploy: - replicas: 1 - update_config: - parallelism: 1 - restart_policy: - condition: on-failure -volumes: - mysql: diff --git a/docker-compose.yml b/docker-compose.yml index e46484f1e4..46ecc81da6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,8 +11,8 @@ services: nginx-proxy: image: nginx:1.21 volumes: - - ./deployments/etc/nginx/nginx.conf:/etc/nginx/nginx.conf - - ./deployments/etc/nginx/conf.d:/etc/nginx/conf.d + - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf + - ./etc/nginx/conf.d:/etc/nginx/conf.d environment: TZ: Asia/Tokyo ports: diff --git a/etc/nginx/conf.d/default.conf b/etc/nginx/conf.d/default.conf new file mode 100644 index 0000000000..0fe20d0290 --- /dev/null +++ b/etc/nginx/conf.d/default.conf @@ -0,0 +1,28 @@ +server { + listen 80 default_server; + client_max_body_size 20M; + root /usr/share/nginx/html; + + gzip on; + gzip_comp_level 4; + gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + if ($time_iso8601 ~ "^(\d{4})-(\d{2})-(\d{2})T(\d{2})") { + set $year $1; + set $month $2; + set $day $3; + set $hour $4; + } + + access_log /var/log/nginx/access.log main; + access_log /var/log/nginx/healthd/application.log.$year-$month-$day-$hour healthd; + + location / { + proxy_pass http://app:3000; + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Host $host:$proxy_port; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} diff --git a/etc/nginx/conf.d/timeout.conf b/etc/nginx/conf.d/timeout.conf new file mode 100644 index 0000000000..1fadc56447 --- /dev/null +++ b/etc/nginx/conf.d/timeout.conf @@ -0,0 +1,3 @@ +proxy_connect_timeout 180; +proxy_send_timeout 180; +proxy_read_timeout 180; diff --git a/etc/nginx/nginx.conf b/etc/nginx/nginx.conf new file mode 100644 index 0000000000..11246998be --- /dev/null +++ b/etc/nginx/nginx.conf @@ -0,0 +1,32 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /var/run/nginx.pid; +worker_rlimit_nofile 32137; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + log_format healthd '$msec"$uri"' + '$status"$request_time"$upstream_response_time"' + '$http_x_forwarded_for'; + + include conf.d/*.conf; + + map $http_upgrade $connection_upgrade { + default "upgrade"; + } + + server_tokens off; +} diff --git a/spec/fixtures/files/heic-image-file.heic b/spec/fixtures/files/heic-image-file.heic new file mode 100644 index 0000000000..91d5dc1497 Binary files /dev/null and b/spec/fixtures/files/heic-image-file.heic differ diff --git a/spec/previewers/heic_previewer_spec.rb b/spec/previewers/heic_previewer_spec.rb new file mode 100644 index 0000000000..64304abdf7 --- /dev/null +++ b/spec/previewers/heic_previewer_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe HeicPreviewer do + it "previews a heic image" do + content_type = "image/heic" + blob = create_file_blob(filename: "heic-image-file.heic", content_type: content_type) + + expect(blob).not_to be_nil + expect(HeicPreviewer.accept?(blob)).to be_truthy # rubocop:disable RSpec/PredicateMatcher + + HeicPreviewer.new(blob).preview({}) do |attachable| + expect(attachable[:content_type]).to eq("image/png") + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 50317642b0..d9547668d5 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -59,4 +59,6 @@ config.filter_rails_from_backtrace! # arbitrary gems may also be filtered via: # config.filter_gems_from_backtrace("gem name") + + config.include ActiveStorageHelpers end diff --git a/spec/support/active_storage_helpers.rb b/spec/support/active_storage_helpers.rb new file mode 100644 index 0000000000..e60a790055 --- /dev/null +++ b/spec/support/active_storage_helpers.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module ActiveStorageHelpers + # ported from https://github.com/rails/rails/blob/4a17b26c6850dd0892dc0b58a6a3f1cce3169593/activestorage/test/test_helper.rb#L52 + def create_file_blob(filename: "image.jpg", content_type: "image/jpeg", metadata: nil) + ActiveStorage::Blob.create_after_upload! io: file_fixture(filename).open, filename: filename, content_type: content_type, metadata: metadata + end +end