diff --git a/lib/gems/pending/util/miq_object_storage.rb b/lib/gems/pending/util/miq_object_storage.rb index 6cb15873f..1f62998c6 100644 --- a/lib/gems/pending/util/miq_object_storage.rb +++ b/lib/gems/pending/util/miq_object_storage.rb @@ -1,12 +1,16 @@ +require 'net/protocol' require 'util/miq_file_storage' class MiqObjectStorage < MiqFileStorage::Interface require 'util/object_storage/miq_s3_storage' require 'util/object_storage/miq_ftp_storage' + require 'util/object_storage/miq_swift_storage' attr_accessor :settings attr_writer :logger + DEFAULT_CHUNKSIZE = Net::BufferedIO::BUFSIZE + def initialize(settings) raise "URI missing" unless settings.key?(:uri) @settings = settings.dup @@ -15,4 +19,33 @@ def initialize(settings) def logger @logger ||= $log.nil? ? :: Logger.new(STDOUT) : $log end + + private + + DONE_READING = "".freeze + def read_single_chunk(chunksize = DEFAULT_CHUNKSIZE) + @buf_left ||= byte_count + return DONE_READING.dup unless @buf_left.nil? || @buf_left.positive? + cur_readsize = if @buf_left.nil? || @buf_left - chunksize >= 0 + chunksize + else + @buf_left + end + buf = source_input.read(cur_readsize) + @buf_left -= chunksize if @buf_left + buf.to_s + end + + def write_single_split_file_for(file_io) + loop do + input_data = read_single_chunk + break if input_data.empty? + file_io.write(input_data) + end + clear_split_vars + end + + def clear_split_vars + @buf_left = nil + end end diff --git a/lib/gems/pending/util/object_storage/miq_swift_storage.rb b/lib/gems/pending/util/object_storage/miq_swift_storage.rb new file mode 100644 index 000000000..13263a20a --- /dev/null +++ b/lib/gems/pending/util/object_storage/miq_swift_storage.rb @@ -0,0 +1,159 @@ +require 'util/miq_object_storage' + +class MiqSwiftStorage < MiqObjectStorage + attr_reader :container_name + + def self.uri_scheme + "swift".freeze + end + + def self.new_with_opts(opts) + new(opts.slice(:uri, :username, :password)) + end + + def initialize(settings) + super(settings) + @bucket_name = URI(@settings[:uri]).host + + raise "username and password are required values!" if @settings[:username].nil? || @settings[:password].nil? + _scheme, _userinfo, @host, @port, _registry, @mount_path, _opaque, query, _fragment = URI.split(URI.encode(@settings[:uri])) + query_params(query) if query + @swift = nil + @username = @settings[:username] + @password = @settings[:password] + @container_name = @mount_path[0] == File::Separator ? @mount_path[1..-1] : @mount_path + end + + def uri_to_object_path(remote_file) + # Strip off the leading "swift://" and the container name from the URI" + # Also remove the leading delimiter. + object_file_with_bucket = URI.split(URI.encode(remote_file))[5] + object_file_with_bucket.split(File::Separator)[2..-1].join(File::Separator) + end + + def upload_single(dest_uri) + # + # Get the remote path, and parse out the bucket name. + # + object_file = uri_to_object_path(dest_uri) + # + # write dump file to swift + # + logger.debug("Writing [#{source_input}] to => Bucket [#{container_name}] using object file name [#{object_file}]") + begin + swift_file = container.files.new(:key => object_file) + params = { + :expects => [201, 202], + :headers => {}, + :request_block => -> { read_single_chunk }, + :idempotent => false, + :method => "PUT", + :path => "#{Fog::OpenStack.escape(swift_file.directory.key)}/#{Fog::OpenStack.escape(swift_file.key)}" + } + # + # Because of how `Fog::OpenStack` (and probably `Fog::Core`) is designed, + # it has hidden the functionality to provide a block for streaming uploads + # that is available out of the box with Excon. + # + # we use .send here because #request is private + # we can't use #put_object (public) directly because it doesn't allow a 202 response code, + # which is what swift responds with when we pass it the :request_block + # (This allows us to stream the response in chunks) + # + swift_file.service.send(:request, params) + clear_split_vars + rescue Excon::Errors::Unauthorized => err + msg = "Access to Swift container #{@container_name} failed due to a bad username or password. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + rescue => err + msg = "Error uploading #{source_input} to Swift container #{@container_name}. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end + end + + def mkdir(_dir) + container + end + + # + # Some calls to Fog::Storage::OpenStack::Directories#get will + # return 'nil', and not return an error. This would cause errors down the + # line in '#upload' or '#download'. + # + # Instead of investigating further, we created a new method that is in charge of + # OpenStack container creation, '#create_container', and that is called from '#container' + # if 'nil' is returned from 'swift.directories.get(container_name)', or in the rescue case + # for 'NotFound' to cover that scenario as well + # + + def container(create_if_missing = true) + @container ||= begin + container = swift.directories.get(container_name) + logger.debug("Swift container [#{container}] found") if container + raise Fog::Storage::OpenStack::NotFound unless container + container + rescue Fog::Storage::OpenStack::NotFound + if create_if_missing + logger.debug("Swift container #{container_name} does not exist. Creating.") + create_container + else + msg = "Swift container #{container_name} does not exist. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end + rescue => err + msg = "Error getting Swift container #{container_name}. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end + end + + private + + def auth_url + URI::Generic.build( + :scheme => @security_protocol == 'non-ssl' ? "http" : "https", + :host => @host, + :port => @port.to_i, + :path => "/#{@api_version}#{@api_version == "v3" ? "/auth" : ".0"}/tokens" + ).to_s + end + + def swift + return @swift if @swift + require 'fog/openstack' + + connection_params = { + :openstack_auth_url => auth_url, + :openstack_username => @username, + :openstack_api_key => @password, + :openstack_project_domain_id => @domain_id, + :openstack_user_domain_id => @domain_id, + :openstack_region => @region, + :connection_options => { :debug_request => true } + } + + @swift = Fog::Storage::OpenStack.new(connection_params) + end + + def create_container + container = swift.directories.create(:key => container_name) + logger.debug("Swift container [#{container_name}] created") + container + rescue => err + msg = "Error creating Swift container #{container_name}. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end + + def download_single(_source, _destination) + raise NotImplementedError, "MiqSwiftStorage.download_single Not Yet Implemented" + end + + def query_params(query_string) + parts = URI.decode_www_form(query_string).to_h + @region, @api_version, @domain_id, @security_protocol = parts.values_at("region", "api_version", "domain_id", "security_protocol") + end +end diff --git a/manageiq-gems-pending.gemspec b/manageiq-gems-pending.gemspec index 27d37f617..96184d2c6 100644 --- a/manageiq-gems-pending.gemspec +++ b/manageiq-gems-pending.gemspec @@ -27,7 +27,9 @@ Gem::Specification.new do |s| s.add_runtime_dependency "aws-sdk", "~> 2.9.7" s.add_runtime_dependency "binary_struct", "~> 2.1" s.add_runtime_dependency "bundler", ">= 1.8.4" # rails-assets requires bundler >= 1.8.4, see: https://rails-assets.org/ + s.add_runtime_dependency "fog-openstack", "~> 0.1.22" s.add_runtime_dependency "linux_admin", "~> 1.0" + s.add_runtime_dependency "mime-types", "~> 3.0" s.add_runtime_dependency "minitar", "~> 0.6" s.add_runtime_dependency "more_core_extensions", "~> 3.4" s.add_runtime_dependency "net-scp", "~> 1.2.1" diff --git a/spec/support/contexts/generated_tmp_files.rb b/spec/support/contexts/generated_tmp_files.rb index 26426ba3f..2300e5e69 100644 --- a/spec/support/contexts/generated_tmp_files.rb +++ b/spec/support/contexts/generated_tmp_files.rb @@ -9,9 +9,20 @@ end after do + # When source_file.unlink is called, it will make it so `source_file.path` + # returns `nil`. Cache it's value incase it hasn't been accessed in the + # tests so we can clear out the generated files properly. + tmp_source_path = source_path + source_file.unlink - Dir["#{source_path.expand_path}.*"].each do |file| + Dir["#{tmp_source_path.expand_path}.*"].each do |file| File.delete(file) end + + if defined?(dest_path) && dest_path.to_s.include?(Dir.tmpdir) + Dir["#{dest_path}*"].each do |file| + File.delete(file) + end + end end end diff --git a/spec/util/miq_file_storage_spec.rb b/spec/util/miq_file_storage_spec.rb index 958520449..420ea90e9 100644 --- a/spec/util/miq_file_storage_spec.rb +++ b/spec/util/miq_file_storage_spec.rb @@ -20,6 +20,18 @@ def opts_for_ftp opts[:uri] = "ftp://example.com/share/path/to/file.txt" end + def opts_for_swift_without_params + opts[:uri] = "swift://example.com/share/path/to/file.txt" + opts[:username] = "user" + opts[:password] = "pass" + end + + def opts_for_swift_with_params + opts[:uri] = "swift://example.com/share/path/to/file.txt?region=foo" + opts[:username] = "user" + opts[:password] = "pass" + end + def opts_for_fakefs opts[:uri] = "foo://example.com/share/path/to/file.txt" end @@ -76,6 +88,18 @@ def opts_for_fakefs include_examples ".with_interface_class implementation", "MiqFtpStorage" end + context "with an swift:// uri" do + before { opts_for_swift_with_params } + + include_examples ".with_interface_class implementation", "MiqSwiftStorage" + end + + context "with an swift:// uri and no query params" do + before { opts_for_swift_without_params } + + include_examples ".with_interface_class implementation", "MiqSwiftStorage" + end + context "with an unknown uri scheme" do before { opts_for_fakefs } diff --git a/spec/util/miq_object_storage_spec.rb b/spec/util/miq_object_storage_spec.rb new file mode 100644 index 000000000..901390549 --- /dev/null +++ b/spec/util/miq_object_storage_spec.rb @@ -0,0 +1,120 @@ +require "fileutils" +require "util/mount/miq_generic_mount_session" +require "util/miq_object_storage" + +class MockLocalFileStorage < MiqObjectStorage + def initialize(source_path = nil, byte_count = 2.megabytes) + @byte_count = byte_count + @source_input = File.open(source_path, "rb") if source_path + @root_dir = Dir.tmpdir + end + + def mkdir(dir) + FileUtils.mkdir_p(File.join(@root_dir, dir)) + end +end + +describe MiqObjectStorage do + describe "#write_single_split_file_for (private)" do + include_context "generated tmp files" + + subject { MockLocalFileStorage.new source_path } + let(:dest_path) { Dir::Tmpname.create("") { |name| name } } + + it "copies file to splits" do + expected_splitfiles = (1..5).map do |suffix| + "#{dest_path}.0000#{suffix}" + end + + expected_splitfiles.each do |file| + subject.send(:write_single_split_file_for, File.open(file, "wb")) + end + + expected_splitfiles.each do |filename| + expect(File.exist?(filename)).to be true + expect(Pathname.new(filename).lstat.size).to eq(2.megabytes) + end + end + + context "with slightly a slightly smaller input file than 10MB" do + let(:tmpfile_size) { 10.megabytes - 1.kilobyte } + subject { MockLocalFileStorage.new source_path, 1.megabyte } + + it "properly chunks the file" do + expected_splitfiles = (1..10).map do |suffix| + "#{dest_path}.%05d" % {:suffix => suffix} + end + + expected_splitfiles.each do |file| + subject.send(:write_single_split_file_for, File.open(file, "wb")) + end + + expected_splitfiles[0, 9].each do |filename| + expect(File.exist?(filename)).to be true + expect(File.size(filename)).to eq(1.megabytes) + end + + last_split = expected_splitfiles.last + expect(File.exist?(last_split)).to be true + expect(File.size(last_split)).to eq(1.megabyte - 1.kilobyte) + end + end + + context "non-split files (byte_count == nil)" do + subject { MockLocalFileStorage.new source_path, byte_count } + let(:byte_count) { nil } + + it "streams the whole file over" do + subject.send(:write_single_split_file_for, File.open(dest_path, "wb")) + expect(File.exist?(dest_path)).to be true + expect(Pathname.new(dest_path).lstat.size).to eq(tmpfile_size) + end + end + end + + describe "#read_single_chunk (private)" do + include_context "generated tmp files" + + subject { MockLocalFileStorage.new source_path } + let(:dest_path) { Dir::Tmpname.create("") { |name| name } } + let(:chunksize) { MockLocalFileStorage::DEFAULT_CHUNKSIZE } + + it "reads 16384 by default" do + chunk_of_data = subject.send(:read_single_chunk) + expect(chunk_of_data).to eq("0" * chunksize) + end + + it "reads the amount of data equal to chunksize when that is passed" do + chunk_of_data = subject.send(:read_single_chunk, 1.kilobyte) + expect(chunk_of_data).to eq("0" * 1.kilobyte) + end + + context "near the end of the split file" do + let(:data_left) { 123 } + let(:penultimate_chunkize) { chunksize - data_left } + + before do + # read an odd amount of data + read_times = 2.megabytes / chunksize + (read_times - 1).times { subject.send(:read_single_chunk) } + subject.send(:read_single_chunk, penultimate_chunkize) + end + + it "reads only what is necessary to finish the split file" do + chunk_of_data = subject.send(:read_single_chunk) + expect(chunk_of_data).to eq("0" * data_left) + end + + it "stops reading until `#clear_split_vars` is called" do + expect(subject.send(:read_single_chunk)).to eq("0" * data_left) + expect(subject.send(:read_single_chunk)).to eq("") + expect(subject.send(:read_single_chunk)).to eq("") + expect(subject.send(:read_single_chunk)).to eq("") + + subject.send(:clear_split_vars) + + expect(subject.send(:read_single_chunk)).to eq("0" * chunksize) + end + end + end +end diff --git a/spec/util/object_storage/miq_swift_storage_spec.rb b/spec/util/object_storage/miq_swift_storage_spec.rb new file mode 100644 index 000000000..248a61cf5 --- /dev/null +++ b/spec/util/object_storage/miq_swift_storage_spec.rb @@ -0,0 +1,69 @@ +require "util/object_storage/miq_swift_storage" + +describe MiqSwiftStorage do + let(:object_storage) { described_class.new(:uri => uri, :username => 'user', :password => 'pass') } + + context "using a uri without query parameters" do + let(:uri) { "swift://foo.com/abc/def" } + + it "#initialize sets the container_name" do + container_name = object_storage.container_name + expect(container_name).to eq("abc/def") + end + + it "#uri_to_object_path returns a new object path" do + result = object_storage.uri_to_object_path(uri) + expect(result).to eq("def") + end + end + + describe "#auth_url (private)" do + context "with non-ssl security protocol" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v3&security_protocol=non-ssl" } + + it "sets the scheme to http" do + expect(URI(object_storage.send(:auth_url)).scheme).to eq("http") + end + + it "sets the host to foo.com" do + expect(URI(object_storage.send(:auth_url)).host).to eq("foo.com") + end + + it "unsets the query string" do + expect(URI(object_storage.send(:auth_url)).query).to eq(nil) + end + end + + context "with ssl security protocol" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v3&security_protocol=ssl" } + + it "sets the scheme to https" do + expect(URI(object_storage.send(:auth_url)).scheme).to eq("https") + end + + it "sets the host to foo.com" do + expect(URI(object_storage.send(:auth_url)).host).to eq("foo.com") + end + + it "unsets the query string" do + expect(URI(object_storage.send(:auth_url)).query).to eq(nil) + end + end + + context "with v3 api version" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v3&security_protocol=ssl" } + + it "sets the path to a v3 path" do + expect(URI(object_storage.send(:auth_url)).path).to eq("/v3/auth/tokens") + end + end + + context "with v2 api version" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v2&security_protocol=ssl" } + + it "sets the path to a v2 path" do + expect(URI(object_storage.send(:auth_url)).path).to eq("/v2.0/tokens") + end + end + end +end