DB Backups to Openstack Swift

lib/gems/pending/util/miq_object_storage.rb

@@ -1,12 +1,16 @@
+require 'net/protocol'
 require 'util/miq_file_storage'
 
 class MiqObjectStorage < MiqFileStorage::Interface
   require 'util/object_storage/miq_s3_storage'
   require 'util/object_storage/miq_ftp_storage'
+  require 'util/object_storage/miq_swift_storage'
 
   attr_accessor :settings
   attr_writer   :logger
 
+  DEFAULT_CHUNKSIZE = Net::BufferedIO::BUFSIZE
+
   def initialize(settings)
     raise "URI missing" unless settings.key?(:uri)
     @settings = settings.dup
@@ -15,4 +19,33 @@ def initialize(settings)
   def logger
     @logger ||= $log.nil? ? :: Logger.new(STDOUT) : $log
   end
+
+  private
+
+  DONE_READING = "".freeze
+  def read_single_chunk(chunksize = DEFAULT_CHUNKSIZE)
+    @buf_left ||= byte_count
+    return DONE_READING.dup unless @buf_left.nil? || @buf_left.positive?
+    cur_readsize = if @buf_left.nil? || @buf_left - chunksize >= 0
+                     chunksize
+                   else
+                     @buf_left
+                   end
+    buf = source_input.read(cur_readsize)
+    @buf_left -= chunksize if @buf_left
+    buf.to_s
+  end
+
+  def write_single_split_file_for(file_io)
+    loop do
+      input_data = read_single_chunk
+      break if input_data.empty?
+      file_io.write(input_data)
+    end
+    clear_split_vars
+  end
+
+  def clear_split_vars
+    @buf_left = nil
+  end
 end

lib/gems/pending/util/object_storage/miq_swift_storage.rb

@@ -0,0 +1,159 @@
+require 'util/miq_object_storage'
+
+class MiqSwiftStorage < MiqObjectStorage
+  attr_reader :container_name
+
+  def self.uri_scheme
+    "swift".freeze
+  end
+
+  def self.new_with_opts(opts)
+    new(opts.slice(:uri, :username, :password))
+  end
+
+  def initialize(settings)
+    super(settings)
+    @bucket_name = URI(@settings[:uri]).host
+
+    raise "username and password are required values!" if @settings[:username].nil? || @settings[:password].nil?
+    _scheme, _userinfo, @host, @port, _registry, @mount_path, _opaque, query, _fragment = URI.split(URI.encode(@settings[:uri]))
+    query_params(query) if query
+    @swift          = nil
+    @username       = @settings[:username]
+    @password       = @settings[:password]
+    @container_name = @mount_path[0] == File::Separator ? @mount_path[1..-1] : @mount_path
+  end
+
+  def uri_to_object_path(remote_file)
+    # Strip off the leading "swift://" and the container name from the URI"
+    # Also remove the leading delimiter.
+    object_file_with_bucket = URI.split(URI.encode(remote_file))[5]
+    object_file_with_bucket.split(File::Separator)[2..-1].join(File::Separator)
+  end
+
+  def upload_single(dest_uri)
+    #
+    # Get the remote path, and parse out the bucket name.
+    #
+    object_file = uri_to_object_path(dest_uri)
+    #
+    # write dump file to swift
+    #
+    logger.debug("Writing [#{source_input}] to => Bucket [#{container_name}] using object file name [#{object_file}]")
+    begin
+      swift_file = container.files.new(:key => object_file)
+      params     = {
+        :expects       => [201, 202],
+        :headers       => {},
+        :request_block => -> { read_single_chunk },
+        :idempotent    => false,
+        :method        => "PUT",
+        :path          => "#{Fog::OpenStack.escape(swift_file.directory.key)}/#{Fog::OpenStack.escape(swift_file.key)}"
+      }
+      #
+      # Because of how `Fog::OpenStack` (and probably `Fog::Core`) is designed,
+      # it has hidden the functionality to provide a block for streaming uploads
+      # that is available out of the box with Excon.
+      #
+      # we use .send here because #request is private
+      # we can't use #put_object (public) directly because it doesn't allow a 202 response code,
+      # which is what swift responds with when we pass it the :request_block
+      # (This allows us to stream the response in chunks)
+      #
+      swift_file.service.send(:request, params)
+      clear_split_vars
+    rescue Excon::Errors::Unauthorized => err
+      msg = "Access to Swift container #{@container_name} failed due to a bad username or password. #{err}"
+      logger.error(msg)
+      raise err, msg, err.backtrace
+    rescue => err
+      msg = "Error uploading #{source_input} to Swift container #{@container_name}. #{err}"
+      logger.error(msg)
+      raise err, msg, err.backtrace
+    end
+  end
+
+  def mkdir(_dir)
+    container
+  end
+
+  #
+  # Some calls to Fog::Storage::OpenStack::Directories#get will
+  # return 'nil', and not return an error.  This would cause errors down the
+  # line in '#upload' or '#download'.
+  #
+  # Instead of investigating further, we created a new method that is in charge of
+  # OpenStack container creation, '#create_container', and that is called from '#container'
+  # if 'nil' is returned from 'swift.directories.get(container_name)', or in the rescue case
+  # for 'NotFound' to cover that scenario as well
+  #
+
+  def container(create_if_missing = true)
+    @container ||= begin
+                     container = swift.directories.get(container_name)
+                     logger.debug("Swift container [#{container}] found") if container
+                     raise Fog::Storage::OpenStack::NotFound unless container
+                     container
+                   rescue Fog::Storage::OpenStack::NotFound
+                     if create_if_missing
+                       logger.debug("Swift container #{container_name} does not exist.  Creating.")
+                       create_container
+                     else
+                       msg = "Swift container #{container_name} does not exist.  #{err}"
+                       logger.error(msg)
+                       raise err, msg, err.backtrace
+                     end
+                   rescue => err
+                     msg = "Error getting Swift container #{container_name}. #{err}"
+                     logger.error(msg)
+                     raise err, msg, err.backtrace
+                   end
+  end
+
+  private
+
+  def auth_url
+    URI::Generic.build(
+      :scheme => @security_protocol == 'non-ssl' ? "http" : "https",
+      :host   => @host,
+      :port   => @port.to_i,
+      :path   => "/#{@api_version}#{@api_version == "v3" ? "/auth" : ".0"}/tokens"
+    ).to_s
+  end
+
+  def swift
+    return @swift if @swift
+    require 'fog/openstack'
+
+    connection_params = {
+      :openstack_auth_url          => auth_url,
+      :openstack_username          => @username,
+      :openstack_api_key           => @password,
+      :openstack_project_domain_id => @domain_id,
+      :openstack_user_domain_id    => @domain_id,
+      :openstack_region            => @region,
+      :connection_options          => { :debug_request => true }
+    }
+
+    @swift = Fog::Storage::OpenStack.new(connection_params)
+  end
+
+  def create_container
+    container = swift.directories.create(:key => container_name)
+    logger.debug("Swift container [#{container_name}] created")
+    container
+  rescue => err
+    msg = "Error creating Swift container #{container_name}. #{err}"
+    logger.error(msg)
+    raise err, msg, err.backtrace
+  end
+
+  def download_single(_source, _destination)
+    raise NotImplementedError, "MiqSwiftStorage.download_single Not Yet Implemented"
+  end
+
+  def query_params(query_string)
+    parts = URI.decode_www_form(query_string).to_h
+    @region, @api_version, @domain_id, @security_protocol = parts.values_at("region", "api_version", "domain_id", "security_protocol")
+  end
+end

manageiq-gems-pending.gemspec

@@ -27,7 +27,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "aws-sdk",                 "~> 2.9.7"
   s.add_runtime_dependency "binary_struct",           "~> 2.1"
   s.add_runtime_dependency "bundler",                 ">= 1.8.4" # rails-assets requires bundler >= 1.8.4, see: https://rails-assets.org/
+  s.add_runtime_dependency "fog-openstack",           "~> 0.1.22"
   s.add_runtime_dependency "linux_admin",             "~> 1.0"
+  s.add_runtime_dependency "mime-types",              "~> 3.0"
   s.add_runtime_dependency "minitar",                 "~> 0.6"
   s.add_runtime_dependency "more_core_extensions",    "~> 3.4"
   s.add_runtime_dependency "net-scp",                 "~> 1.2.1"

spec/support/contexts/generated_tmp_files.rb

@@ -9,9 +9,20 @@
   end
 
   after do
+    # When source_file.unlink is called, it will make it so `source_file.path`
+    # returns `nil`.  Cache it's value incase it hasn't been accessed in the
+    # tests so we can clear out the generated files properly.
+    tmp_source_path = source_path
+
     source_file.unlink
-    Dir["#{source_path.expand_path}.*"].each do |file|
+    Dir["#{tmp_source_path.expand_path}.*"].each do |file|
       File.delete(file)
     end
+
+    if defined?(dest_path) && dest_path.to_s.include?(Dir.tmpdir)
+      Dir["#{dest_path}*"].each do |file|
+        File.delete(file)
+      end
+    end
   end
 end

spec/util/miq_file_storage_spec.rb

@@ -20,6 +20,18 @@ def opts_for_ftp
     opts[:uri] = "ftp://example.com/share/path/to/file.txt"
   end
 
+  def opts_for_swift_without_params
+    opts[:uri]      = "swift://example.com/share/path/to/file.txt"
+    opts[:username] = "user"
+    opts[:password] = "pass"
+  end
+
+  def opts_for_swift_with_params
+    opts[:uri]      = "swift://example.com/share/path/to/file.txt?region=foo"
+    opts[:username] = "user"
+    opts[:password] = "pass"
+  end
+
   def opts_for_fakefs
     opts[:uri] = "foo://example.com/share/path/to/file.txt"
   end
@@ -76,6 +88,18 @@ def opts_for_fakefs
       include_examples ".with_interface_class implementation", "MiqFtpStorage"
     end
 
+    context "with an swift:// uri" do
+      before { opts_for_swift_with_params }
+
+      include_examples ".with_interface_class implementation", "MiqSwiftStorage"
+    end
+
+    context "with an swift:// uri and no query params" do
+      before { opts_for_swift_without_params }
+
+      include_examples ".with_interface_class implementation", "MiqSwiftStorage"
+    end
+
     context "with an unknown uri scheme" do
       before { opts_for_fakefs }