diff --git a/modules/storages/app/common/storages/errors.rb b/modules/storages/app/common/storages/errors.rb index 04efd72676ca..3e1b2cefa937 100644 --- a/modules/storages/app/common/storages/errors.rb +++ b/modules/storages/app/common/storages/errors.rb @@ -34,6 +34,8 @@ class BaseError < StandardError; end class ResolverStandardError < BaseError; end + class PollingRequired < BaseError; end + class MissingContract < ResolverStandardError; end class OperationNotSupported < ResolverStandardError; end diff --git a/modules/storages/app/common/storages/peripherals/nextcloud.rb b/modules/storages/app/common/storages/peripherals/nextcloud_registry.rb similarity index 97% rename from modules/storages/app/common/storages/peripherals/nextcloud.rb rename to modules/storages/app/common/storages/peripherals/nextcloud_registry.rb index 0d5a40cc18b1..059a769e2761 100644 --- a/modules/storages/app/common/storages/peripherals/nextcloud.rb +++ b/modules/storages/app/common/storages/peripherals/nextcloud_registry.rb @@ -30,7 +30,7 @@ module Storages module Peripherals - Nextcloud = Dry::Container::Namespace.new('nextcloud') do + NextcloudRegistry = Dry::Container::Namespace.new('nextcloud') do namespace('queries') do register(:download_link, StorageInteraction::Nextcloud::DownloadLinkQuery) register(:file_ids, StorageInteraction::Nextcloud::FileIdsQuery) diff --git a/modules/storages/app/common/storages/peripherals/one_drive.rb b/modules/storages/app/common/storages/peripherals/one_drive_registry.rb similarity index 93% rename from modules/storages/app/common/storages/peripherals/one_drive.rb rename to modules/storages/app/common/storages/peripherals/one_drive_registry.rb index 399df0b46c39..55307a1d2e91 100644 --- a/modules/storages/app/common/storages/peripherals/one_drive.rb +++ b/modules/storages/app/common/storages/peripherals/one_drive_registry.rb @@ -30,7 +30,7 @@ module Storages module Peripherals - OneDrive = Dry::Container::Namespace.new('one_drive') do + OneDriveRegistry = Dry::Container::Namespace.new('one_drive') do namespace('queries') do register(:download_link, StorageInteraction::OneDrive::DownloadLinkQuery) register(:files, StorageInteraction::OneDrive::FilesQuery) @@ -43,6 +43,7 @@ module Peripherals end namespace('commands') do + register(:copy_template_folder, StorageInteraction::OneDrive::CopyTemplateFolderCommand) register(:create_folder, StorageInteraction::OneDrive::CreateFolderCommand) register(:delete_folder, StorageInteraction::OneDrive::DeleteFolderCommand) register(:rename_file, StorageInteraction::OneDrive::RenameFileCommand) diff --git a/modules/storages/app/common/storages/peripherals/registry.rb b/modules/storages/app/common/storages/peripherals/registry.rb index af92eee674bf..4912485a9fc7 100644 --- a/modules/storages/app/common/storages/peripherals/registry.rb +++ b/modules/storages/app/common/storages/peripherals/registry.rb @@ -44,7 +44,7 @@ def call(container, key) config.resolver = Resolver.new end - Registry.import Nextcloud - Registry.import OneDrive + Registry.import NextcloudRegistry + Registry.import OneDriveRegistry end end diff --git a/modules/storages/app/services/projects/copy/storage_project_folders_dependent_service.rb b/modules/storages/app/services/projects/copy/storage_project_folders_dependent_service.rb index 8dc78a329588..4f62f4e06e97 100644 --- a/modules/storages/app/services/projects/copy/storage_project_folders_dependent_service.rb +++ b/modules/storages/app/services/projects/copy/storage_project_folders_dependent_service.rb @@ -46,6 +46,12 @@ def source_count def copy_dependency(*) return unless state.copied_project_storages + GoodJob::Batches.enqueue(on_success: NotifyCopyCompletedJob, project_storages: state.copied_project_storages) do + state.copied_project_storages.each do |project_storages| + ::Storages::CopyProjectFoldersJob.enqueue(project_storages[:source], project_storages[:target]) + end + end + state.copied_project_storages.each do |copied_project_storage| source = copied_project_storage[:source] target = copied_project_storage[:target] diff --git a/modules/storages/app/services/storages/project_storages/copy_project_folders_service.rb b/modules/storages/app/services/storages/project_storages/copy_project_folders_service.rb new file mode 100644 index 000000000000..edb899a23ac4 --- /dev/null +++ b/modules/storages/app/services/storages/project_storages/copy_project_folders_service.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) 2012-2024 the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Storages + module ProjectStorages + class CopyProjectFoldersService + # We might need the User too + def self.call(source_id:, target_id:) + new(source_id, target_id).call + end + + def initialize(source_id, target_id) + @source = get_project_storage(source_id) + @target = get_project_storage(target_id) + end + + def call + return ServiceResult.success if @source.project_folder_inactive? + return update_target(@source.project_folder_id) if @source.project_folder_manual? + + copy_result = copy_project_folder.on_failure { |failed_result| return failed_result }.result + + update_target(copy_result[:id]) if copy_result[:id] + + ServiceResult.failure(result: copy_result[:url], errors: :polling_required) + end + + private + + def copy_project_folder + Peripherals::Registry + .resolve("#{@source.storage.short_provider_type}.commands.copy_template_folder") + .call(storage: @source.storage, + source_path: @source.project_folder_location, + destination_path: @target.managed_project_folder_path) + end + + def update_target(project_folder_id) + ProjectStorages::UpdateService + .new(user: User.system, model: @target) + .call({ project_folder_id:, project_folder_mode: @source.project_folder_mode }) + end + + def get_project_storage(id) + ProjectStorage.includes(:project, :storage).find(id) + end + end + end +end diff --git a/modules/storages/app/workers/storages/copy_project_folders_job.rb b/modules/storages/app/workers/storages/copy_project_folders_job.rb new file mode 100644 index 000000000000..5f28078abe0c --- /dev/null +++ b/modules/storages/app/workers/storages/copy_project_folders_job.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +#-- copyright +#++ + +module Storages + class CopyProjectFoldersJob < ApplicationJob + # include GoodJob::ActiveJobExtensions::Batches + + retry_on Errors::PollingRequired, wait: 3, attempts: :unlimited + # discard_on HTTPX::HTTPError + + def perform(user_id:, source_id:, target_id:, work_package_map:) + target = ProjectStorage.find(target_id) + source = ProjectStorage.find(source_id) + user = User.find(user_id) + new_work_package_map = work_package_map.transform_keys(&:to_i) + + project_folder_result = if polling? + results_from_polling + else + initiate_project_folder_copy(source, target) + end + + project_folder_id = project_folder_result.on_failure { |failed_result| return failed_result }.result + + # TODO: Do Something when this fails + ProjectStorages::UpdateService.new(user:, model: target) + .call(project_folder_id:, project_folder_mode: source.project_folder_mode) + + # We only get here on a successful execution + create_target_file_links(source, target, new_work_package_map, user) + end + + private + + def create_target_file_links(source, target, work_package_map, user) + source_file_links = FileLink + .includes(:creator) + .where(container_id: work_package_map.keys, container_type: "WorkPackage") + + return create_unmanaged_file_links(source_file_links, work_package_map, user) if source.project_folder_manual? + + target_files = Peripherals::Registry + .resolve("#{source.storage.short_provider_type}.queries.folder_files_file_ids_deep_query") + .call(storage: source.storage, folder: Peripherals::ParentFolder.new(target.project_folder_location)) + .result + + source_files = Peripherals::Registry + .resolve("#{source.storage.short_provider_type}.queries.files_info") + .call(storage: source.storage, user:, file_ids: source_file_links.pluck(:origin_id)) + .result + + source_location_map = source_files.to_h { |info| [info.id, info.location] } + + source_file_links.find_each do |source_link| + attributes = source_link.dup.attributes + + attributes['creator_id'] = user.id + attributes['container_id'] = work_package_map[source_link.container_id] + + source_link_location = source_location_map[source_link.origin_id] + target_link_location = source_link_location.gsub(source.managed_project_folder_path, target.managed_project_folder_path) + + attributes['origin_id'] = target_files[target_link_location] + + FileLinks::CreateService.new(user:).call(attributes) + end + end + + def create_unmanaged_file_links(source_file_links, work_package_map, user) + source_file_links.find_each do |source_file_link| + attributes = source_file_link.dup.attributes + + attributes['creator_id'] = user.id + attributes['container_id'] = work_package_map[source_file_link.container_id] + + # TODO: Do something when this fails + FileLinks::CreateService.new(user:).call(attributes) + end + end + + def initiate_project_folder_copy(source, target) + return ServiceResult.success if source.project_folder_inactive? + return ServiceResult.success(result: source.project_folder_id) if source.project_folder_manual? + + copy_result = issue_command(source, target).on_failure { |failed_result| return failed_result }.result + return ServiceResult.success(result: copy_result[:id]) if copy_result[:id] + + Thread.current[job_id] = copy_result[:url] + raise Errors::PollingRequired, "#{job_id} Storage requires polling" + end + + def issue_command(source, target) + Peripherals::Registry + .resolve("#{source.storage.short_provider_type}.commands.copy_template_folder") + .call(storage: source.storage, + source_path: source.project_folder_location, + destination_path: target.managed_project_folder_path) + end + + def polling? + !!Thread.current[job_id] + end + + def results_from_polling + # TODO: Maybe Transform this in a Query + response = OpenProject.httpx.get(Thread.current[job_id]).json(symbolize_keys: true) + + raise(Errors::PollingRequired, "#{job_id} Polling not completed yet") if response[:status] != 'completed' + + Thread.current[job_id] = nil + ServiceResult.success(result: response[:resourceId]) + end + end +end diff --git a/modules/storages/spec/services/storages/project_storages/copy_project_folders_service_spec.rb b/modules/storages/spec/services/storages/project_storages/copy_project_folders_service_spec.rb new file mode 100644 index 000000000000..ea8c1bef6cfa --- /dev/null +++ b/modules/storages/spec/services/storages/project_storages/copy_project_folders_service_spec.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) 2012-2024 the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require 'spec_helper' +require_module_spec_helper + +RSpec.describe Storages::ProjectStorages::CopyProjectFoldersService, :webmock do + let(:storage) { create(:nextcloud_storage, :as_automatically_managed) } + let(:target) { create(:project_storage, storage:) } + let(:system_user) { create(:system) } + + subject(:service) { described_class } + + context "with automatically managed project folders" do + let(:source) { create(:project_storage, :as_automatically_managed, storage:) } + + it 'updates the target project storage to point to newly copied remote folder' do + Storages::Peripherals::Registry + .stub("#{source.storage.short_provider_type}.commands.copy_template_folder", + ->(args) do + # Validating the arguments ensure that the call is correctly made + expect(args[:storage]).to eq(source.storage) + expect(args[:source_path]).to eq(source.project_folder_location) + expect(args[:destination_path]).to eq(target.managed_project_folder_path) + + # Return a success for the provider copy with no polling required + ServiceResult.success(result: { id: 'newly_created_remote_folder', url: 'https://resource.url' }) + end) + + expect(service.call(source_id: source.id, target_id: target.id)).to be_success + + target.reload + expect(target.project_folder_mode).to eq(source.project_folder_mode) + expect(target.project_folder_id).to eq('newly_created_remote_folder') + end + + it 'if polling is required, returns an error with the polling url' do + Storages::Peripherals::Registry + .stub("#{source.storage.short_provider_type}.commands.copy_template_folder", + ->(args) do + # Validating the arguments ensure that the call is correctly made + expect(args[:storage]).to eq(source.storage) + expect(args[:source_path]).to eq(source.project_folder_location) + expect(args[:destination_path]).to eq(target.managed_project_folder_path) + + # Return a success for the provider copy with no polling required + ServiceResult.success(result: { id: nil, url: 'https://polling.url.de/cool/subresources' }) + end) + + result = service.call(source_id: source.id, target_id: target.id) + + expect(result).to be_failure + expect(result.result).to eq('https://polling.url.de/cool/subresources') + expect(result.errors).to eq(:polling_required) + end + end + + context "with manually managed project folders" do + let(:source) { create(:project_storage, project_folder_id: 'this_is_a_unique_id', project_folder_mode: 'manual') } + + it "succeeds" do + expect(service.call(source_id: source.id, target_id: target.id)).to be_success + end + + it "updates to the target project storage to point to the same project_folder_id than the source" do + expect { service.call(source_id: source.id, target_id: target.id) } + .to change { target.reload.project_folder_id } + .to(source.project_folder_id) + .and(change { target.reload.project_folder_mode } + .to(source.project_folder_mode)) + end + end + + context "with non-managed project folders" do + let(:source) { create(:project_storage, project_folder_id: 'this_is_a_unique_id') } + + it "succeeds" do + expect(service.call(source_id: source.id, target_id: target.id)).to be_success + end + + it "doesn't require any updates to the target project storage" do + expect { service.call(source_id: source.id, target_id: target.id) }.not_to change(target, :project_folder_id) + end + end +end diff --git a/modules/storages/spec/workers/storages/copy_project_folders_job_spec.rb b/modules/storages/spec/workers/storages/copy_project_folders_job_spec.rb new file mode 100644 index 000000000000..a10ea634e09d --- /dev/null +++ b/modules/storages/spec/workers/storages/copy_project_folders_job_spec.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +#-- copyright +#++ + +require 'spec_helper' +require_module_spec_helper + +RSpec.describe Storages::CopyProjectFoldersJob, :job, :webmock do + include ActiveJob::TestHelper + + let(:storage) { create(:nextcloud_storage, :as_automatically_managed) } + let(:source) { create(:project_storage, :as_automatically_managed, storage:) } + let(:user) { create(:admin) } + + let(:source_work_packages) { create_list(:work_package, 4, project: source.project) } + + let(:target) { create(:project_storage, storage: source.storage) } + let(:target_work_packages) { create_list(:work_package, 4, project: target.project) } + + let(:work_package_map) do + source_work_packages + .pluck(:id) + .map(&:to_s) + .zip(target_work_packages.pluck(:id)) + .to_h + end + + let(:polling_url) { 'https://polling.url.de/cool/subresources' } + + let(:target_deep_file_ids) do + source_file_links.each_with_object({}) do |fl, hash| + hash["#{target.managed_project_folder_path}#{fl.name}"] = "RANDOM_ID_#{fl.hash}" + end + end + + let(:source_file_links) { source_work_packages.map { |wp| create(:file_link, container: wp, storage:) } } + let(:source_file_infos) do + source_file_links.map do |fl| + Storages::StorageFileInfo.new( + status: 'ok', + status_code: 200, + id: fl.origin_id, + name: fl.name, + location: "#{source.managed_project_folder_path}#{fl.name}" + ) + end + end + + before do + # Limit the number of retries on tests + described_class.retry_on Storages::Errors::PollingRequired, wait: 1, attempts: 3 + source_file_links + end + + describe "non-automatic managed folders" do + let(:inverted_wp_map) { work_package_map.invert } + + before do + source.update(project_folder_mode: 'manual', project_folder_id: 'awesome-folder') + source.reload + end + + it 'updates the target project storage project_folder_id to match the source' do + perform_enqueued_jobs(only: described_class) do + described_class.perform_now(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end + + target.reload + expect(target.project_folder_id).to eq(source.project_folder_id) + end + + it 'copies all the file link info on the corresponding work_package' do + perform_enqueued_jobs(only: described_class) do + described_class.perform_now(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end + + WorkPackage.includes(:file_links).where(id: work_package_map.values).find_each do |target_wp| + expect(target_wp.file_links.count).to eq(1) + + file_link = target_wp.file_links.first + source_file_link = source_file_links.find do |fl| + fl.container_id == inverted_wp_map[target_wp.id].to_i + end + + expect(file_link.origin_name).to eq(source_file_link.origin_name) + expect(file_link.origin_id).to eq(source_file_link.origin_id) + end + end + end + + # rubocop:disable Lint/UnusedBlockArgument + describe "managed project folders" do + before do + Storages::Peripherals::Registry + .stub("#{storage.short_provider_type}.queries.folder_files_file_ids_deep_query", ->(storage:, folder:) { + ServiceResult.success(result: target_deep_file_ids) + }) + + Storages::Peripherals::Registry + .stub("#{storage.short_provider_type}.queries.files_info", ->(storage:, user:, file_ids:) { + ServiceResult.success(result: source_file_infos) + }) + + Storages::Peripherals::Registry + .stub("#{storage.short_provider_type}.commands.copy_template_folder", ->(storage:, source_path:, destination_path:) { + ServiceResult.success(result: { id: 'copied-folder', url: 'resource-url' }) + }) + end + + it 'copies the folders from source to target' do + perform_enqueued_jobs(only: described_class) do + described_class.perform_now(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end + + target.reload + expect(target.project_folder_mode).to eq(source.project_folder_mode) + expect(target.project_folder_id).to eq('copied-folder') + end + + it 'creates the file links pointing to the newly copied files' do + perform_enqueued_jobs(only: described_class) do + described_class.perform_now(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end + + Storages::FileLink.where(container: target_work_packages).find_each do |file_link| + expect(file_link.origin_id).to eq(target_deep_file_ids["#{target.managed_project_folder_path}#{file_link.name}"]) + end + end + end + + context "when the storage requires polling" do + before do + Storages::Peripherals::Registry + .stub("#{storage.short_provider_type}.commands.copy_template_folder", ->(storage:, source_path:, destination_path:) { + ServiceResult.success(result: { id: nil, url: polling_url }) + }) + + Storages::Peripherals::Registry + .stub("#{storage.short_provider_type}.queries.folder_files_file_ids_deep_query", ->(storage:, folder:) { + ServiceResult.success(result: target_deep_file_ids) + }) + + Storages::Peripherals::Registry + .stub("#{storage.short_provider_type}.queries.files_info", ->(storage:, user:, file_ids:) { + ServiceResult.success(result: source_file_infos) + }) + end + + it 'raises a Storages::Errors::PollingRequired' do + perform_enqueued_jobs(only: described_class) do + expect do + described_class.perform_now(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end.to raise_error Storages::Errors::PollingRequired + end + end + + it 'stores the polling url on the current thread' do + job = described_class.new + + perform_enqueued_jobs(only: described_class) do + expect do + job.perform(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end.to raise_error Storages::Errors::PollingRequired + end + + expect(Thread.current[job.job_id]).to eq(polling_url) + end + + context 'when the polling completes' do + let(:copy_incomplete_response) do + { operation: "ItemCopy", percentageComplete: 27.8, status: "inProgress" }.to_json + end + + let(:copy_complete_response) do + { percentageComplete: 100.0, resourceId: "01MOWKYVJML57KN2ANMBA3JZJS2MBGC7KM", status: "completed" }.to_json + end + + before do + stub_request(:get, polling_url) + .and_return( + { status: 202, body: copy_incomplete_response, headers: { 'Content-Type' => 'application/json' } }, + { status: 202, body: copy_complete_response, headers: { 'Content-Type' => 'application/json' } } + ) + end + + it 'updates the storages' do + perform_enqueued_jobs(only: described_class) do + described_class.perform_now(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end + + target.reload + expect(target.project_folder_mode).to eq(source.project_folder_mode) + expect(target.project_folder_id).to eq('01MOWKYVJML57KN2ANMBA3JZJS2MBGC7KM') + end + + it 'handles re-enqueues and polling' do + perform_enqueued_jobs(only: described_class) do + described_class.perform_now(source_id: source.id, target_id: target.id, work_package_map:, user_id: user.id) + end + + performed_job = ActiveJob::Base.queue_adapter.performed_jobs.find { |jobs| jobs['job_class'] == described_class.to_s } + expect(performed_job['exception_executions']['[Storages::Errors::PollingRequired]']).to eq(2) + expect(performed_job['executions']).to eq(1) + end + end + end + # rubocop:enable Lint/UnusedBlockArgument +end