diff --git a/app/jobs/model_scan_job.rb b/app/jobs/model_scan_job.rb index f281cabfe..f53437fd1 100644 --- a/app/jobs/model_scan_job.rb +++ b/app/jobs/model_scan_job.rb @@ -14,7 +14,7 @@ def file_list(model_path, library, include_all_subfolders: false) pattern ) end - library.list_files(glob).uniq + library.list_files(glob) end def perform(model_id, include_all_subfolders: false) diff --git a/app/jobs/process_uploaded_file_job.rb b/app/jobs/process_uploaded_file_job.rb index 8b72f6708..4b80a9f2e 100644 --- a/app/jobs/process_uploaded_file_job.rb +++ b/app/jobs/process_uploaded_file_job.rb @@ -43,7 +43,7 @@ def unzip(model, uploaded_file) Archive::Reader.open_filename(archive.path, strip_components: strip) do |reader| reader.each_entry do |entry| next if !entry.file? || entry.size > SiteSettings.max_file_extract_size - next if hidden?(entry.pathname) + next if SiteSettings.ignored_file?(entry.pathname) Dir.chdir(tmpdir) do reader.extract(entry, Archive::EXTRACT_SECURE) model.model_files.create(filename: entry.pathname, attachment: File.open(entry.pathname)) @@ -54,10 +54,6 @@ def unzip(model, uploaded_file) end end - def hidden?(pathname) - (File.split(pathname) - ["."]).any? { |x| x.starts_with? "." } - end - def count_common_path_components(archive) # Generate full list of directories in the archive paths = [] diff --git a/app/models/library.rb b/app/models/library.rb index cab769913..c5ffed2d7 100644 --- a/app/models/library.rb +++ b/app/models/library.rb @@ -116,7 +116,7 @@ def self.register_all_storage end def list_files(pattern, flags = 0) - case storage_service + files = case storage_service when "filesystem" Dir.glob(pattern, flags, base: Shellwords.escape(path)).filter { |x| File.file?(File.join(path, x)) } when "s3" @@ -129,6 +129,8 @@ def list_files(pattern, flags = 0) else raise "Invalid storage service: #{storage_service}" end + # Filter out files that should be ignored + files.uniq.reject { |str| SiteSettings.ignored_file?(str) } end def has_file?(path) diff --git a/app/models/site_settings.rb b/app/models/site_settings.rb index d251c53b8..7c7cf264d 100644 --- a/app/models/site_settings.rb +++ b/app/models/site_settings.rb @@ -41,6 +41,17 @@ def self.federation_enabled? Rails.application.config.manyfold_features[:federation] end + def self.ignored_file?(pathname) + @@patterns ||= [ + /^\.[^\.]+/, # Hidden files starting with . + /.*\/@eaDir\/.*/, # Synology temp files + /__MACOSX/ # MACOS resource forks + ] + (File.split(pathname) - ["."]).any? do |path_component| + @@patterns.any? { |pattern| path_component =~ pattern } + end + end + module UserDefaults RENDERER = { grid_width: 200, diff --git a/spec/jobs/process_uploaded_file_job_spec.rb b/spec/jobs/process_uploaded_file_job_spec.rb index 0c35e3add..54c99ead4 100644 --- a/spec/jobs/process_uploaded_file_job_spec.rb +++ b/spec/jobs/process_uploaded_file_job_spec.rb @@ -1,28 +1,6 @@ require "rails_helper" RSpec.describe ProcessUploadedFileJob do - context "when detecting hidden files" do - it "detects normal files" do - expect(described_class.new.send(:hidden?, "test.stl")).to be false - end - - it "detects normal files in subfolders" do - expect(described_class.new.send(:hidden?, "test/test.stl")).to be false - end - - it "detects hidden files" do - expect(described_class.new.send(:hidden?, ".test.stl")).to be true - end - - it "detects hidden files in subfolders" do - expect(described_class.new.send(:hidden?, "test/.test.stl")).to be true - end - - it "detects normal files in hidden subfolders" do - expect(described_class.new.send(:hidden?, ".test/test.stl")).to be true - end - end - context "when counting common path prefixes" do it "returns zero if there are no directories at all" do expect(described_class.new.send(:count_common_elements, [])).to eq 0 diff --git a/spec/models/site_settings_spec.rb b/spec/models/site_settings_spec.rb new file mode 100644 index 000000000..0ee215c2d --- /dev/null +++ b/spec/models/site_settings_spec.rb @@ -0,0 +1,26 @@ +require "rails_helper" + +RSpec.describe SiteSettings do + context "when detecting ignored files" do + %w[ + test.stl + test/test.stl + ].each do |pathname| + it "accepts `#{pathname}`" do + expect(described_class.send(:ignored_file?, pathname)).to be false + end + end + + %w[ + .test.stl + test/.test.stl + .test/test.stl + model/@eaDir/test.png/SYNOPHOTO_THUMB_S.png + model/__MACOSX + ].each do |pathname| + it "ignores `#{pathname}`" do + expect(described_class.send(:ignored_file?, pathname)).to be true + end + end + end +end