Merge pull request #2674 from manyfold3d/unzip-in-cache-folder

Improve performance and disk usage during upload processing

app/jobs/process_uploaded_file_job.rb

@@ -45,18 +45,22 @@ def perform(library_id, uploaded_file, owner: nil, creator_id: nil, collection_i
 
   def unzip(model, uploaded_file)
     Shrine.with_file(uploaded_file) do |archive|
-      Dir.mktmpdir do |tmpdir|
-        strip = count_common_path_components(archive)
-        Archive::Reader.open_filename(archive.path, strip_components: strip) do |reader|
-          reader.each_entry do |entry|
-            next if !entry.file? || entry.size > SiteSettings.max_file_extract_size
-            next if SiteSettings.ignored_file?(entry.pathname)
-            filename = entry.pathname # Stored because pathname gets mutated by the extract and we want the original
-            reader.extract(entry, Archive::EXTRACT_SECURE, destination: tmpdir)
-            model.model_files.create(filename: filename, attachment: File.open(entry.pathname))
-          end
+      tmpdir = Shrine.find_storage(:cache).directory.join(SecureRandom.uuid)
+      tmpdir.mkdir
+      strip = count_common_path_components(archive)
+      Archive::Reader.open_filename(archive.path, strip_components: strip) do |reader|
+        reader.each_entry do |entry|
+          next if !entry.file? || entry.size > SiteSettings.max_file_extract_size
+          next if SiteSettings.ignored_file?(entry.pathname)
+          filename = entry.pathname # Stored because pathname gets mutated by the extract and we want the original
+          reader.extract(entry, Archive::EXTRACT_SECURE, destination: tmpdir.to_s)
+          model.model_files.create(filename: filename, attachment: File.open(entry.pathname))
+          # Clean up file
+          File.delete(entry.pathname) if File.exist?(entry.pathname)
         end
       end
+      # Clean up temp folder
+      Dir.rmdir(tmpdir) if Dir.empty?(tmpdir)
     end
   end
 

config/initializers/shrine.rb

@@ -8,11 +8,22 @@
 Shrine.plugin :rack_response
 
 Shrine.storages = {
-  cache: Shrine::Storage::FileSystem.new("tmp/cache")
+  cache: Shrine::Storage::FileSystem.new("tmp/shrine")
 }
 
 Rails.application.config.after_initialize do
   Library.register_all_storage
+
+  begin
+    upload_options = {cache: {move: true}}
+    Library.all.map do |l|
+      upload_options[l.storage_key.to_sym] = {move: true} if l.storage_service == "filesystem"
+    end
+    Shrine.plugin :upload_options, **upload_options unless Rails.env.test?
+  rescue ActiveRecord::StatementInvalid, NameError
+    nil # migrations probably haven't run yet to create library table
+  end
+
   begin
     Sidekiq.set_schedule("sweep", {every: "1h", class: "CacheSweepJob"})
   rescue RedisClient::CannotConnectError